2024-11-10 04:21:01,242 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 04:21:01,256 main DEBUG Took 0.011993 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 04:21:01,256 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 04:21:01,257 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 04:21:01,258 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 04:21:01,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,266 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 04:21:01,281 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,283 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,285 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,286 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,287 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,288 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,289 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,290 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,290 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,291 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,292 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,292 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:21:01,293 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,293 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 04:21:01,294 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:21:01,296 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 04:21:01,297 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 04:21:01,298 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 04:21:01,299 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 04:21:01,299 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 04:21:01,308 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 04:21:01,310 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 04:21:01,312 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 04:21:01,312 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 04:21:01,312 main DEBUG createAppenders(={Console}) 2024-11-10 04:21:01,313 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-10 04:21:01,313 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 04:21:01,314 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-10 04:21:01,314 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 04:21:01,314 main DEBUG OutputStream closed 2024-11-10 04:21:01,315 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 04:21:01,315 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 04:21:01,315 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-10 04:21:01,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 04:21:01,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 04:21:01,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 04:21:01,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 04:21:01,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 04:21:01,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 04:21:01,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 04:21:01,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 04:21:01,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 04:21:01,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 04:21:01,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 04:21:01,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 04:21:01,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 04:21:01,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 04:21:01,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 04:21:01,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 04:21:01,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 04:21:01,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 04:21:01,393 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 04:21:01,393 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-10 04:21:01,393 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 04:21:01,394 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-10T04:21:01,628 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126 2024-11-10 04:21:01,631 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 04:21:01,631 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T04:21:01,640 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-11-10T04:21:01,674 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=17, ProcessCount=11, AvailableMemoryMB=8120 2024-11-10T04:21:01,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:21:01,693 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552, deleteOnExit=true 2024-11-10T04:21:01,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:21:01,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/test.cache.data in system properties and HBase conf 2024-11-10T04:21:01,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:21:01,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:21:01,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:21:01,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:21:01,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:21:01,792 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T04:21:01,889 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:21:01,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:21:01,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:21:01,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:21:01,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:21:01,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:21:01,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:21:01,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:21:01,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:21:01,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:21:01,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:21:01,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:21:01,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:21:01,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:21:01,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:21:02,434 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:21:02,773 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T04:21:02,857 INFO [Time-limited test {}] log.Log(170): Logging initialized @2401ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T04:21:02,936 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:21:02,999 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:21:03,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:21:03,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:21:03,020 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:21:03,033 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:21:03,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:21:03,037 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:21:03,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/java.io.tmpdir/jetty-localhost-35071-hadoop-hdfs-3_4_1-tests_jar-_-any-17298793073906345149/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:21:03,271 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:35071} 2024-11-10T04:21:03,272 INFO [Time-limited test {}] server.Server(415): Started @2817ms 2024-11-10T04:21:03,303 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:21:03,703 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:21:03,713 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:21:03,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:21:03,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:21:03,715 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:21:03,719 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:21:03,720 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:21:03,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/java.io.tmpdir/jetty-localhost-35515-hadoop-hdfs-3_4_1-tests_jar-_-any-16953674621538071880/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:21:03,871 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:35515} 2024-11-10T04:21:03,871 INFO [Time-limited test {}] server.Server(415): Started @3416ms 2024-11-10T04:21:03,933 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:21:04,069 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:21:04,074 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:21:04,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:21:04,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:21:04,079 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:21:04,079 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:21:04,080 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:21:04,206 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/java.io.tmpdir/jetty-localhost-41081-hadoop-hdfs-3_4_1-tests_jar-_-any-16053364565362727857/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:21:04,207 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:41081} 2024-11-10T04:21:04,208 INFO [Time-limited test {}] server.Server(415): Started @3753ms 2024-11-10T04:21:04,211 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:21:04,434 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data2/current/BP-1998714310-172.17.0.2-1731212462530/current, will proceed with Du for space computation calculation, 2024-11-10T04:21:04,434 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data4/current/BP-1998714310-172.17.0.2-1731212462530/current, will proceed with Du for space computation calculation, 2024-11-10T04:21:04,434 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data1/current/BP-1998714310-172.17.0.2-1731212462530/current, will proceed with Du for space computation calculation, 2024-11-10T04:21:04,435 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data3/current/BP-1998714310-172.17.0.2-1731212462530/current, will proceed with Du for space computation calculation, 2024-11-10T04:21:04,494 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:21:04,495 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:21:04,567 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb82d848a6e21da84 with lease ID 0xc1288646e8dcc4f3: Processing first storage report for DS-fdbcd528-9eaf-4704-a903-125c097c9193 from datanode DatanodeRegistration(127.0.0.1:44523, datanodeUuid=35d6de79-48d2-4c36-94c1-86f9b18b9263, infoPort=37651, infoSecurePort=0, ipcPort=40773, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530) 2024-11-10T04:21:04,569 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb82d848a6e21da84 with lease ID 0xc1288646e8dcc4f3: from storage DS-fdbcd528-9eaf-4704-a903-125c097c9193 node DatanodeRegistration(127.0.0.1:44523, datanodeUuid=35d6de79-48d2-4c36-94c1-86f9b18b9263, infoPort=37651, infoSecurePort=0, ipcPort=40773, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T04:21:04,569 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc2df3709a51dc619 with lease ID 0xc1288646e8dcc4f4: Processing first storage report for DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3 from datanode DatanodeRegistration(127.0.0.1:39761, datanodeUuid=8d606198-1d31-4f46-995e-fc52a703af32, infoPort=41385, infoSecurePort=0, ipcPort=39405, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530) 2024-11-10T04:21:04,569 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2df3709a51dc619 with lease ID 0xc1288646e8dcc4f4: from storage DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3 node DatanodeRegistration(127.0.0.1:39761, datanodeUuid=8d606198-1d31-4f46-995e-fc52a703af32, infoPort=41385, infoSecurePort=0, ipcPort=39405, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:21:04,570 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb82d848a6e21da84 with lease ID 0xc1288646e8dcc4f3: Processing first storage report for DS-b698c62a-d4b0-4cc2-b695-f9af28858133 from datanode DatanodeRegistration(127.0.0.1:44523, datanodeUuid=35d6de79-48d2-4c36-94c1-86f9b18b9263, infoPort=37651, infoSecurePort=0, ipcPort=40773, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530) 2024-11-10T04:21:04,570 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb82d848a6e21da84 with lease ID 0xc1288646e8dcc4f3: from storage DS-b698c62a-d4b0-4cc2-b695-f9af28858133 node DatanodeRegistration(127.0.0.1:44523, datanodeUuid=35d6de79-48d2-4c36-94c1-86f9b18b9263, infoPort=37651, infoSecurePort=0, ipcPort=40773, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:21:04,570 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc2df3709a51dc619 with lease ID 0xc1288646e8dcc4f4: Processing first storage report for DS-5da4571d-137b-4c13-b866-410aa8e6b20d from datanode DatanodeRegistration(127.0.0.1:39761, datanodeUuid=8d606198-1d31-4f46-995e-fc52a703af32, infoPort=41385, infoSecurePort=0, ipcPort=39405, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530) 2024-11-10T04:21:04,570 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2df3709a51dc619 with lease ID 0xc1288646e8dcc4f4: from storage DS-5da4571d-137b-4c13-b866-410aa8e6b20d node DatanodeRegistration(127.0.0.1:39761, datanodeUuid=8d606198-1d31-4f46-995e-fc52a703af32, infoPort=41385, infoSecurePort=0, ipcPort=39405, storageInfo=lv=-57;cid=testClusterID;nsid=227649432;c=1731212462530), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:21:04,626 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126 2024-11-10T04:21:04,704 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/zookeeper_0, clientPort=52254, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:21:04,713 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52254 2024-11-10T04:21:04,727 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:04,731 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:04,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:21:04,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:21:05,380 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce with version=8 2024-11-10T04:21:05,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase-staging 2024-11-10T04:21:05,472 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T04:21:05,724 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:21:05,736 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:21:05,736 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:21:05,740 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:21:05,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:21:05,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:21:05,898 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:21:05,959 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T04:21:05,967 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T04:21:05,971 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:21:05,998 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32460 (auto-detected) 2024-11-10T04:21:05,999 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T04:21:06,018 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42827 2024-11-10T04:21:06,045 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42827 connecting to ZooKeeper ensemble=127.0.0.1:52254 2024-11-10T04:21:06,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428270x0, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:21:06,089 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42827-0x101906947960000 connected 2024-11-10T04:21:06,127 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:06,130 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:06,140 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:21:06,145 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce, hbase.cluster.distributed=false 2024-11-10T04:21:06,172 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:21:06,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42827 2024-11-10T04:21:06,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42827 2024-11-10T04:21:06,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42827 2024-11-10T04:21:06,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42827 2024-11-10T04:21:06,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42827 2024-11-10T04:21:06,306 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:21:06,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:21:06,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:21:06,309 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:21:06,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:21:06,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:21:06,312 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:21:06,315 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:21:06,316 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43425 2024-11-10T04:21:06,319 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43425 connecting to ZooKeeper ensemble=127.0.0.1:52254 2024-11-10T04:21:06,320 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:06,326 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:06,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434250x0, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:21:06,337 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434250x0, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:21:06,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43425-0x101906947960001 connected 2024-11-10T04:21:06,342 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:21:06,352 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:21:06,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:21:06,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:21:06,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43425 2024-11-10T04:21:06,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43425 2024-11-10T04:21:06,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43425 2024-11-10T04:21:06,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43425 2024-11-10T04:21:06,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43425 2024-11-10T04:21:06,382 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:42827 2024-11-10T04:21:06,383 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:06,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:21:06,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:21:06,392 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:06,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:21:06,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:06,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:06,414 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:21:06,415 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,42827,1731212465525 from backup master directory 2024-11-10T04:21:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:21:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:21:06,419 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:21:06,420 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:06,422 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T04:21:06,423 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T04:21:06,480 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase.id] with ID: 0c915281-58f6-4167-adfe-37c8575ab0a3 2024-11-10T04:21:06,480 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/.tmp/hbase.id 2024-11-10T04:21:06,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:21:06,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:21:06,493 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/.tmp/hbase.id]:[hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase.id] 2024-11-10T04:21:06,540 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:06,546 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:21:06,566 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-10T04:21:06,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:06,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:06,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:21:06,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:21:06,603 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:21:06,605 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:21:06,611 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:21:06,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:21:06,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:21:06,667 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store 2024-11-10T04:21:06,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:21:06,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:21:06,694 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T04:21:06,698 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:21:06,699 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:21:06,700 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:21:06,700 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:21:06,702 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:21:06,702 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:21:06,702 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:21:06,704 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212466699Disabling compacts and flushes for region at 1731212466699Disabling writes for close at 1731212466702 (+3 ms)Writing region close event to WAL at 1731212466702Closed at 1731212466702 2024-11-10T04:21:06,706 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/.initializing 2024-11-10T04:21:06,706 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/WALs/d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:06,730 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C42827%2C1731212465525, suffix=, logDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/WALs/d1d9e3766cb1,42827,1731212465525, archiveDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/oldWALs, maxLogs=10 2024-11-10T04:21:06,739 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C42827%2C1731212465525.1731212466735 2024-11-10T04:21:06,765 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/WALs/d1d9e3766cb1,42827,1731212465525/d1d9e3766cb1%2C42827%2C1731212465525.1731212466735 2024-11-10T04:21:06,776 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37651:37651),(127.0.0.1/127.0.0.1:41385:41385)] 2024-11-10T04:21:06,777 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:21:06,778 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:21:06,781 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,782 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:21:06,853 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:06,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:06,857 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:21:06,861 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:06,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:21:06,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:21:06,865 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:06,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:21:06,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:21:06,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:06,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:21:06,871 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,875 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,877 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,882 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,883 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,886 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:21:06,889 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:21:06,894 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:21:06,895 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=816335, jitterRate=0.03802485764026642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:21:06,901 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212466795Initializing all the Stores at 1731212466797 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212466798 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212466798Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212466799 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212466799Cleaning up temporary data from old regions at 1731212466883 (+84 ms)Region opened successfully at 1731212466901 (+18 ms) 2024-11-10T04:21:06,902 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:21:06,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33cdd114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:21:06,974 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:21:06,986 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:21:06,986 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:21:06,991 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:21:06,992 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T04:21:06,997 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-10T04:21:06,997 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:21:07,021 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:21:07,029 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:21:07,032 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:21:07,034 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:21:07,036 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:21:07,038 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:21:07,041 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:21:07,045 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:21:07,047 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:21:07,048 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:21:07,050 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:21:07,067 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:21:07,069 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:21:07,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:21:07,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:21:07,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,076 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,42827,1731212465525, sessionid=0x101906947960000, setting cluster-up flag (Was=false) 2024-11-10T04:21:07,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,096 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:21:07,097 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:07,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,110 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:21:07,112 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:07,119 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:21:07,171 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(746): ClusterId : 0c915281-58f6-4167-adfe-37c8575ab0a3 2024-11-10T04:21:07,174 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:21:07,180 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:21:07,180 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:21:07,183 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:21:07,184 DEBUG [RS:0;d1d9e3766cb1:43425 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735ef924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:21:07,203 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:43425 2024-11-10T04:21:07,204 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:21:07,206 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:21:07,206 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:21:07,206 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:21:07,209 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,42827,1731212465525 with port=43425, startcode=1731212466265 2024-11-10T04:21:07,216 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:21:07,220 DEBUG [RS:0;d1d9e3766cb1:43425 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:21:07,224 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:21:07,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,42827,1731212465525 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:21:07,238 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:21:07,238 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:21:07,238 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:21:07,238 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:21:07,238 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:21:07,238 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,239 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:21:07,239 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,243 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212497242 2024-11-10T04:21:07,244 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:21:07,244 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:21:07,245 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:21:07,245 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:21:07,250 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:21:07,250 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:21:07,251 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:21:07,251 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:21:07,253 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:07,253 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:21:07,252 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,259 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:21:07,260 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:21:07,261 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:21:07,268 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:21:07,268 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:21:07,272 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212467270,5,FailOnTimeoutGroup] 2024-11-10T04:21:07,275 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212467272,5,FailOnTimeoutGroup] 2024-11-10T04:21:07,275 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,276 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:21:07,278 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,280 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:21:07,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:21:07,290 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:21:07,291 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce 2024-11-10T04:21:07,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34041, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:21:07,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:21:07,318 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42827 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:07,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:21:07,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:21:07,322 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42827 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:07,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:21:07,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:21:07,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:07,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:07,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:21:07,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:21:07,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:07,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:07,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:21:07,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:21:07,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:07,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:07,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:21:07,341 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce 2024-11-10T04:21:07,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:21:07,341 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45907 2024-11-10T04:21:07,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:07,341 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:21:07,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:07,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:21:07,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740 2024-11-10T04:21:07,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740 2024-11-10T04:21:07,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:21:07,347 DEBUG [RS:0;d1d9e3766cb1:43425 {}] zookeeper.ZKUtil(111): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:07,347 WARN [RS:0;d1d9e3766cb1:43425 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:21:07,347 INFO [RS:0;d1d9e3766cb1:43425 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:21:07,347 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:07,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:21:07,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:21:07,350 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,43425,1731212466265] 2024-11-10T04:21:07,350 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:21:07,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:21:07,361 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:21:07,361 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804433, jitterRate=0.022890210151672363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:21:07,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212467321Initializing all the Stores at 1731212467323 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212467324 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212467324Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212467324Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212467324Cleaning up temporary data from old regions at 1731212467349 (+25 ms)Region opened successfully at 1731212467364 (+15 ms) 2024-11-10T04:21:07,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:21:07,364 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:21:07,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:21:07,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:21:07,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:21:07,366 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:21:07,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212467364Disabling compacts and flushes for region at 1731212467364Disabling writes for close at 1731212467365 (+1 ms)Writing region close event to WAL at 1731212467365Closed at 1731212467366 (+1 ms) 2024-11-10T04:21:07,369 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:21:07,370 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:21:07,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:21:07,379 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:21:07,384 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:21:07,388 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:21:07,393 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:21:07,398 INFO [RS:0;d1d9e3766cb1:43425 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:21:07,399 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,400 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:21:07,408 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:21:07,409 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,410 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,410 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,410 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,410 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,411 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,411 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:21:07,411 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,411 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,412 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,412 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,412 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,412 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:21:07,413 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:21:07,413 DEBUG [RS:0;d1d9e3766cb1:43425 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:21:07,414 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,414 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,415 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,415 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,415 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,415 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43425,1731212466265-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:21:07,442 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:21:07,444 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43425,1731212466265-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,444 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,444 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.Replication(171): d1d9e3766cb1,43425,1731212466265 started 2024-11-10T04:21:07,466 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:07,466 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,43425,1731212466265, RpcServer on d1d9e3766cb1/172.17.0.2:43425, sessionid=0x101906947960001 2024-11-10T04:21:07,467 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:21:07,468 DEBUG [RS:0;d1d9e3766cb1:43425 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:07,468 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,43425,1731212466265' 2024-11-10T04:21:07,468 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:21:07,469 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:21:07,470 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:21:07,470 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:21:07,470 DEBUG [RS:0;d1d9e3766cb1:43425 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:07,471 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,43425,1731212466265' 2024-11-10T04:21:07,471 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:21:07,471 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:21:07,472 DEBUG [RS:0;d1d9e3766cb1:43425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:21:07,472 INFO [RS:0;d1d9e3766cb1:43425 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:21:07,472 INFO [RS:0;d1d9e3766cb1:43425 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:21:07,539 WARN [d1d9e3766cb1:42827 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:21:07,581 INFO [RS:0;d1d9e3766cb1:43425 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C43425%2C1731212466265, suffix=, logDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265, archiveDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs, maxLogs=32 2024-11-10T04:21:07,584 INFO [RS:0;d1d9e3766cb1:43425 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212467583 2024-11-10T04:21:07,593 INFO [RS:0;d1d9e3766cb1:43425 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212467583 2024-11-10T04:21:07,596 DEBUG [RS:0;d1d9e3766cb1:43425 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41385:41385),(127.0.0.1/127.0.0.1:37651:37651)] 2024-11-10T04:21:07,791 DEBUG [d1d9e3766cb1:42827 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:21:07,803 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:07,810 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,43425,1731212466265, state=OPENING 2024-11-10T04:21:07,817 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:21:07,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:21:07,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:21:07,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:21:07,822 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:21:07,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,43425,1731212466265}] 2024-11-10T04:21:08,000 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:21:08,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54719, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:21:08,014 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:21:08,015 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:21:08,018 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C43425%2C1731212466265.meta, suffix=.meta, logDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265, archiveDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs, maxLogs=32 2024-11-10T04:21:08,020 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.meta.1731212468020.meta 2024-11-10T04:21:08,029 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.meta.1731212468020.meta 2024-11-10T04:21:08,030 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41385:41385),(127.0.0.1/127.0.0.1:37651:37651)] 2024-11-10T04:21:08,030 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:21:08,032 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:21:08,035 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:21:08,040 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:21:08,045 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:21:08,045 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:21:08,045 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:21:08,045 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:21:08,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:21:08,050 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:21:08,050 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:08,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:08,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:21:08,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:21:08,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:08,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:08,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:21:08,055 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:21:08,055 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:08,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:08,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:21:08,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:21:08,057 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:08,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:21:08,058 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:21:08,059 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740 2024-11-10T04:21:08,062 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740 2024-11-10T04:21:08,064 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:21:08,064 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:21:08,065 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:21:08,067 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:21:08,069 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=774618, jitterRate=-0.015023380517959595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:21:08,069 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:21:08,071 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212468046Writing region info on filesystem at 1731212468046Initializing all the Stores at 1731212468048 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212468048Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212468048Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212468048Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212468048Cleaning up temporary data from old regions at 1731212468064 (+16 ms)Running coprocessor post-open hooks at 1731212468069 (+5 ms)Region opened successfully at 1731212468070 (+1 ms) 2024-11-10T04:21:08,078 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212467991 2024-11-10T04:21:08,090 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:21:08,090 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:21:08,092 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:08,094 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,43425,1731212466265, state=OPEN 2024-11-10T04:21:08,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:21:08,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:21:08,100 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:21:08,100 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:21:08,100 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:08,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:21:08,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,43425,1731212466265 in 277 msec 2024-11-10T04:21:08,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:21:08,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 732 msec 2024-11-10T04:21:08,116 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:21:08,116 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:21:08,140 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:21:08,142 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,43425,1731212466265, seqNum=-1] 2024-11-10T04:21:08,164 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:21:08,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54641, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:21:08,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0350 sec 2024-11-10T04:21:08,187 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212468186, completionTime=-1 2024-11-10T04:21:08,191 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:21:08,191 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:21:08,226 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:21:08,226 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212528226 2024-11-10T04:21:08,227 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731212588227 2024-11-10T04:21:08,227 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-11-10T04:21:08,231 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42827,1731212465525-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:08,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42827,1731212465525-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:08,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42827,1731212465525-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:08,234 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:42827, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:08,235 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:08,235 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:08,242 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:21:08,270 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.850sec 2024-11-10T04:21:08,272 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:21:08,274 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:21:08,275 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:21:08,276 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:21:08,276 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:21:08,277 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42827,1731212465525-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:21:08,278 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42827,1731212465525-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:21:08,288 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:21:08,289 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:21:08,290 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42827,1731212465525-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:21:08,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c60f60f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:21:08,389 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T04:21:08,389 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T04:21:08,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42827,-1 for getting cluster id 2024-11-10T04:21:08,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:21:08,407 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0c915281-58f6-4167-adfe-37c8575ab0a3' 2024-11-10T04:21:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:21:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0c915281-58f6-4167-adfe-37c8575ab0a3" 2024-11-10T04:21:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e9c12a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:21:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42827,-1] 2024-11-10T04:21:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:21:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:21:08,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:21:08,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae5d858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:21:08,423 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:21:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,43425,1731212466265, seqNum=-1] 2024-11-10T04:21:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:21:08,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:21:08,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:08,456 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:21:08,464 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T04:21:08,469 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:21:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42827,1731212465525 2024-11-10T04:21:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38798fc2 2024-11-10T04:21:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:21:08,481 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:21:08,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42827 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T04:21:08,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42827 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T04:21:08,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42827 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:21:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42827 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-10T04:21:08,498 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:21:08,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42827 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-10T04:21:08,500 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:08,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:21:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:21:08,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741835_1011 (size=389) 2024-11-10T04:21:08,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741835_1011 (size=389) 2024-11-10T04:21:08,549 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d90b6241a0d0a0b09c9951c9f565effe, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce 2024-11-10T04:21:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741836_1012 (size=72) 2024-11-10T04:21:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741836_1012 (size=72) 2024-11-10T04:21:08,962 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:21:08,962 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing d90b6241a0d0a0b09c9951c9f565effe, disabling compactions & flushes 2024-11-10T04:21:08,962 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:08,963 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:08,963 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. after waiting 0 ms 2024-11-10T04:21:08,963 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:08,963 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:08,963 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for d90b6241a0d0a0b09c9951c9f565effe: Waiting for close lock at 1731212468962Disabling compacts and flushes for region at 1731212468962Disabling writes for close at 1731212468963 (+1 ms)Writing region close event to WAL at 1731212468963Closed at 1731212468963 2024-11-10T04:21:08,965 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:21:08,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731212468965"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212468965"}]},"ts":"1731212468965"} 2024-11-10T04:21:08,976 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:21:08,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:21:08,982 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212468979"}]},"ts":"1731212468979"} 2024-11-10T04:21:08,986 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-10T04:21:08,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d90b6241a0d0a0b09c9951c9f565effe, ASSIGN}] 2024-11-10T04:21:08,991 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d90b6241a0d0a0b09c9951c9f565effe, ASSIGN 2024-11-10T04:21:08,993 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d90b6241a0d0a0b09c9951c9f565effe, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,43425,1731212466265; forceNewPlan=false, retain=false 2024-11-10T04:21:09,144 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d90b6241a0d0a0b09c9951c9f565effe, regionState=OPENING, regionLocation=d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:09,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d90b6241a0d0a0b09c9951c9f565effe, ASSIGN because future has completed 2024-11-10T04:21:09,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d90b6241a0d0a0b09c9951c9f565effe, server=d1d9e3766cb1,43425,1731212466265}] 2024-11-10T04:21:09,312 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:09,313 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d90b6241a0d0a0b09c9951c9f565effe, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:21:09,313 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,314 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:21:09,314 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,314 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,316 INFO [StoreOpener-d90b6241a0d0a0b09c9951c9f565effe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,319 INFO [StoreOpener-d90b6241a0d0a0b09c9951c9f565effe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d90b6241a0d0a0b09c9951c9f565effe columnFamilyName info 2024-11-10T04:21:09,319 DEBUG [StoreOpener-d90b6241a0d0a0b09c9951c9f565effe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:21:09,320 INFO [StoreOpener-d90b6241a0d0a0b09c9951c9f565effe-1 {}] regionserver.HStore(327): Store=d90b6241a0d0a0b09c9951c9f565effe/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:21:09,320 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,322 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,323 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,323 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,324 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,326 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,330 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:21:09,331 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d90b6241a0d0a0b09c9951c9f565effe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=809150, jitterRate=0.028887465596199036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:21:09,331 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:09,332 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d90b6241a0d0a0b09c9951c9f565effe: Running coprocessor pre-open hook at 1731212469314Writing region info on filesystem at 1731212469314Initializing all the Stores at 1731212469316 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212469316Cleaning up temporary data from old regions at 1731212469324 (+8 ms)Running coprocessor post-open hooks at 1731212469331 (+7 ms)Region opened successfully at 1731212469332 (+1 ms) 2024-11-10T04:21:09,334 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe., pid=6, masterSystemTime=1731212469305 2024-11-10T04:21:09,338 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:09,339 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:09,340 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d90b6241a0d0a0b09c9951c9f565effe, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,43425,1731212466265 2024-11-10T04:21:09,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d90b6241a0d0a0b09c9951c9f565effe, server=d1d9e3766cb1,43425,1731212466265 because future has completed 2024-11-10T04:21:09,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T04:21:09,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d90b6241a0d0a0b09c9951c9f565effe, server=d1d9e3766cb1,43425,1731212466265 in 195 msec 2024-11-10T04:21:09,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T04:21:09,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d90b6241a0d0a0b09c9951c9f565effe, ASSIGN in 363 msec 2024-11-10T04:21:09,358 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:21:09,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212469358"}]},"ts":"1731212469358"} 2024-11-10T04:21:09,362 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-10T04:21:09,364 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:21:09,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 875 msec 2024-11-10T04:21:13,569 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:21:13,613 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:21:13,614 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-10T04:21:15,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T04:21:15,956 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T04:21:15,957 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-10T04:21:15,957 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T04:21:15,958 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:21:15,958 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T04:21:15,958 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T04:21:15,958 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T04:21:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:21:18,549 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-10T04:21:18,553 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-10T04:21:18,560 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-10T04:21:18,561 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:21:18,561 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212478561 2024-11-10T04:21:18,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:18,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:18,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:18,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:18,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:18,572 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212467583 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212478561 2024-11-10T04:21:18,573 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37651:37651),(127.0.0.1/127.0.0.1:41385:41385)] 2024-11-10T04:21:18,573 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212467583 is not closed yet, will try archiving it next time 2024-11-10T04:21:18,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741833_1009 (size=451) 2024-11-10T04:21:18,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741833_1009 (size=451) 2024-11-10T04:21:18,576 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212467583 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs/d1d9e3766cb1%2C43425%2C1731212466265.1731212467583 2024-11-10T04:21:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe., hostname=d1d9e3766cb1,43425,1731212466265, seqNum=2] 2024-11-10T04:21:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43425 {}] regionserver.HRegion(8855): Flush requested on d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:30,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d90b6241a0d0a0b09c9951c9f565effe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:21:30,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/117dbc11c66b4ed5acf741f183875173 is 1080, key is row0001/info:/1731212478585/Put/seqid=0 2024-11-10T04:21:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741838_1014 (size=12509) 2024-11-10T04:21:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741838_1014 (size=12509) 2024-11-10T04:21:30,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/117dbc11c66b4ed5acf741f183875173 2024-11-10T04:21:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/117dbc11c66b4ed5acf741f183875173 as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173 2024-11-10T04:21:30,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173, entries=7, sequenceid=11, filesize=12.2 K 2024-11-10T04:21:30,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d90b6241a0d0a0b09c9951c9f565effe in 144ms, sequenceid=11, compaction requested=false 2024-11-10T04:21:30,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d90b6241a0d0a0b09c9951c9f565effe: 2024-11-10T04:21:34,623 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:21:38,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212498626 2024-11-10T04:21:38,835 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:21:38,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:38,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:38,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:38,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:38,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:38,836 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212478561 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212498626 2024-11-10T04:21:38,837 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41385:41385),(127.0.0.1/127.0.0.1:37651:37651)] 2024-11-10T04:21:38,837 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212478561 is not closed yet, will try archiving it next time 2024-11-10T04:21:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741837_1013 (size=12399) 2024-11-10T04:21:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741837_1013 (size=12399) 2024-11-10T04:21:39,041 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:41,245 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:43,449 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:45,653 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43425 {}] regionserver.HRegion(8855): Flush requested on d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:21:45,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d90b6241a0d0a0b09c9951c9f565effe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:21:45,855 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:45,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/e5555798a28b4deba417677fdd1b621a is 1080, key is row0008/info:/1731212492616/Put/seqid=0 2024-11-10T04:21:45,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741840_1016 (size=12509) 2024-11-10T04:21:45,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741840_1016 (size=12509) 2024-11-10T04:21:45,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/e5555798a28b4deba417677fdd1b621a 2024-11-10T04:21:45,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/e5555798a28b4deba417677fdd1b621a as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/e5555798a28b4deba417677fdd1b621a 2024-11-10T04:21:45,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/e5555798a28b4deba417677fdd1b621a, entries=7, sequenceid=21, filesize=12.2 K 2024-11-10T04:21:46,094 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:46,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d90b6241a0d0a0b09c9951c9f565effe in 441ms, sequenceid=21, compaction requested=false 2024-11-10T04:21:46,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d90b6241a0d0a0b09c9951c9f565effe: 2024-11-10T04:21:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-11-10T04:21:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:21:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173 because midkey is the same as first or last row 2024-11-10T04:21:47,857 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:48,581 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T04:21:48,581 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T04:21:50,061 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:50,063 WARN [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:50,064 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C43425%2C1731212466265:(num 1731212498626) roll requested 2024-11-10T04:21:50,064 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212510064 2024-11-10T04:21:50,273 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:21:50,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:50,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:50,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:50,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:50,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:21:50,274 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212498626 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212510064 2024-11-10T04:21:50,275 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37651:37651),(127.0.0.1/127.0.0.1:41385:41385)] 2024-11-10T04:21:50,275 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212498626 is not closed yet, will try archiving it next time 2024-11-10T04:21:50,275 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212478561 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs/d1d9e3766cb1%2C43425%2C1731212466265.1731212478561 2024-11-10T04:21:50,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741839_1015 (size=7739) 2024-11-10T04:21:50,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741839_1015 (size=7739) 2024-11-10T04:21:52,265 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:21:54,314 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d90b6241a0d0a0b09c9951c9f565effe, had cached 0 bytes from a total of 25018 2024-11-10T04:21:54,469 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:21:56,673 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:21:58,877 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:00,879 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T04:22:00,880 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212520880 2024-11-10T04:22:04,623 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:22:05,889 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:05,891 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:05,891 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C43425%2C1731212466265:(num 1731212520880) roll requested 2024-11-10T04:22:05,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:05,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:05,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:05,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:05,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:05,892 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212510064 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212520880 2024-11-10T04:22:05,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37651:37651),(127.0.0.1/127.0.0.1:41385:41385)] 2024-11-10T04:22:05,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212510064 is not closed yet, will try archiving it next time 2024-11-10T04:22:05,893 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 2024-11-10T04:22:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741841_1017 (size=4753) 2024-11-10T04:22:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741841_1017 (size=4753) 2024-11-10T04:22:10,897 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:10,897 WARN [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43425 {}] regionserver.HRegion(8855): Flush requested on d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:22:10,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d90b6241a0d0a0b09c9951c9f565effe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:22:10,903 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:10,904 WARN [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:12,898 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T04:22:15,899 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:15,900 WARN [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK], DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK]] 2024-11-10T04:22:15,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:15,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:15,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:15,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:15,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:15,901 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212520880 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 2024-11-10T04:22:15,902 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41385:41385),(127.0.0.1/127.0.0.1:37651:37651)] 2024-11-10T04:22:15,902 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212520880 is not closed yet, will try archiving it next time 2024-11-10T04:22:15,903 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C43425%2C1731212466265:(num 1731212525893) roll requested 2024-11-10T04:22:15,903 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212535903 2024-11-10T04:22:15,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741842_1018 (size=1569) 2024-11-10T04:22:15,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741842_1018 (size=1569) 2024-11-10T04:22:15,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/09d861f6daf649cb93d25fea195ed9fd is 1080, key is row0015/info:/1731212507655/Put/seqid=0 2024-11-10T04:22:15,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741844_1020 (size=12509) 2024-11-10T04:22:15,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741844_1020 (size=12509) 2024-11-10T04:22:15,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/09d861f6daf649cb93d25fea195ed9fd 2024-11-10T04:22:15,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/09d861f6daf649cb93d25fea195ed9fd as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/09d861f6daf649cb93d25fea195ed9fd 2024-11-10T04:22:15,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/09d861f6daf649cb93d25fea195ed9fd, entries=7, sequenceid=31, filesize=12.2 K 2024-11-10T04:22:20,911 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:22:20,911 WARN [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:22:20,934 INFO [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:22:20,934 WARN [FSHLog-0-hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce-prefix:d1d9e3766cb1,43425,1731212466265 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39761,DS-0f5b9286-a66e-4bc7-a03d-54a095d009c3,DISK], DatanodeInfoWithStorage[127.0.0.1:44523,DS-fdbcd528-9eaf-4704-a903-125c097c9193,DISK]] 2024-11-10T04:22:20,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d90b6241a0d0a0b09c9951c9f565effe in 10037ms, sequenceid=31, compaction requested=true 2024-11-10T04:22:20,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d90b6241a0d0a0b09c9951c9f565effe: 2024-11-10T04:22:20,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-11-10T04:22:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:22:20,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173 because midkey is the same as first or last row 2024-11-10T04:22:20,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,935 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212535903 2024-11-10T04:22:20,935 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41385:41385),(127.0.0.1/127.0.0.1:37651:37651)] 2024-11-10T04:22:20,936 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 is not closed yet, will try archiving it next time 2024-11-10T04:22:20,936 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212498626 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs/d1d9e3766cb1%2C43425%2C1731212466265.1731212498626 2024-11-10T04:22:20,936 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C43425%2C1731212466265:(num 1731212540936) roll requested 2024-11-10T04:22:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d90b6241a0d0a0b09c9951c9f565effe:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:22:20,936 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212540936 2024-11-10T04:22:20,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741843_1019 (size=438) 2024-11-10T04:22:20,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741843_1019 (size=438) 2024-11-10T04:22:20,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:22:20,938 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:22:20,939 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212510064 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs/d1d9e3766cb1%2C43425%2C1731212466265.1731212510064 2024-11-10T04:22:20,941 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212520880 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs/d1d9e3766cb1%2C43425%2C1731212466265.1731212520880 2024-11-10T04:22:20,941 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:22:20,943 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.HStore(1541): d90b6241a0d0a0b09c9951c9f565effe/info is initiating minor compaction (all files) 2024-11-10T04:22:20,943 INFO [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d90b6241a0d0a0b09c9951c9f565effe/info in TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:22:20,944 INFO [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173, hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/e5555798a28b4deba417677fdd1b621a, hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/09d861f6daf649cb93d25fea195ed9fd] into tmpdir=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp, totalSize=36.6 K 2024-11-10T04:22:20,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,945 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 117dbc11c66b4ed5acf741f183875173, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731212478585 2024-11-10T04:22:20,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,945 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212535903 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212540936 2024-11-10T04:22:20,946 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5555798a28b4deba417677fdd1b621a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731212492616 2024-11-10T04:22:20,946 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d861f6daf649cb93d25fea195ed9fd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731212507655 2024-11-10T04:22:20,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741845_1021 (size=93) 2024-11-10T04:22:20,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741845_1021 (size=93) 2024-11-10T04:22:20,949 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 is not closed yet, will try archiving it next time 2024-11-10T04:22:20,949 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212535903 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs/d1d9e3766cb1%2C43425%2C1731212466265.1731212535903 2024-11-10T04:22:20,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41385:41385),(127.0.0.1/127.0.0.1:37651:37651)] 2024-11-10T04:22:20,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 is not closed yet, will try archiving it next time 2024-11-10T04:22:20,957 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43425%2C1731212466265.1731212540956 2024-11-10T04:22:20,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:20,969 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212540936 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212540956 2024-11-10T04:22:20,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741846_1022 (size=1258) 2024-11-10T04:22:20,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741846_1022 (size=1258) 2024-11-10T04:22:20,972 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 is not closed yet, will try archiving it next time 2024-11-10T04:22:20,976 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37651:37651),(127.0.0.1/127.0.0.1:41385:41385)] 2024-11-10T04:22:20,976 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 is not closed yet, will try archiving it next time 2024-11-10T04:22:20,983 INFO [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d90b6241a0d0a0b09c9951c9f565effe#info#compaction#3 average throughput is 21.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:22:20,984 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/c10d99410a4c48c2b4c613fa86aeea76 is 1080, key is row0001/info:/1731212478585/Put/seqid=0 2024-11-10T04:22:20,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741848_1024 (size=27710) 2024-11-10T04:22:20,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741848_1024 (size=27710) 2024-11-10T04:22:21,000 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/c10d99410a4c48c2b4c613fa86aeea76 as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/c10d99410a4c48c2b4c613fa86aeea76 2024-11-10T04:22:21,016 INFO [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d90b6241a0d0a0b09c9951c9f565effe/info of d90b6241a0d0a0b09c9951c9f565effe into c10d99410a4c48c2b4c613fa86aeea76(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:22:21,016 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d90b6241a0d0a0b09c9951c9f565effe: 2024-11-10T04:22:21,018 INFO [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe., storeName=d90b6241a0d0a0b09c9951c9f565effe/info, priority=13, startTime=1731212540936; duration=0sec 2024-11-10T04:22:21,018 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T04:22:21,018 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:22:21,018 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/c10d99410a4c48c2b4c613fa86aeea76 because midkey is the same as first or last row 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/c10d99410a4c48c2b4c613fa86aeea76 because midkey is the same as first or last row 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/c10d99410a4c48c2b4c613fa86aeea76 because midkey is the same as first or last row 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:22:21,019 DEBUG [RS:0;d1d9e3766cb1:43425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d90b6241a0d0a0b09c9951c9f565effe:info 2024-11-10T04:22:21,339 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/WALs/d1d9e3766cb1,43425,1731212466265/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs/d1d9e3766cb1%2C43425%2C1731212466265.1731212525893 2024-11-10T04:22:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43425 {}] regionserver.HRegion(8855): Flush requested on d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:22:32,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d90b6241a0d0a0b09c9951c9f565effe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:22:32,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/64164f6e4761430faa73c4a380be3a99 is 1080, key is row0022/info:/1731212540958/Put/seqid=0 2024-11-10T04:22:32,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741849_1025 (size=12509) 2024-11-10T04:22:32,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741849_1025 (size=12509) 2024-11-10T04:22:32,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/64164f6e4761430faa73c4a380be3a99 2024-11-10T04:22:33,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/64164f6e4761430faa73c4a380be3a99 as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/64164f6e4761430faa73c4a380be3a99 2024-11-10T04:22:33,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/64164f6e4761430faa73c4a380be3a99, entries=7, sequenceid=42, filesize=12.2 K 2024-11-10T04:22:33,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d90b6241a0d0a0b09c9951c9f565effe in 34ms, sequenceid=42, compaction requested=false 2024-11-10T04:22:33,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d90b6241a0d0a0b09c9951c9f565effe: 2024-11-10T04:22:33,016 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-11-10T04:22:33,016 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:22:33,016 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/c10d99410a4c48c2b4c613fa86aeea76 because midkey is the same as first or last row 2024-11-10T04:22:34,624 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:22:39,314 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d90b6241a0d0a0b09c9951c9f565effe, had cached 0 bytes from a total of 40219 2024-11-10T04:22:40,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:22:40,992 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:22:40,993 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:22:40,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:40,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:40,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:22:40,998 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:22:40,998 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=888866681, stopped=false 2024-11-10T04:22:40,999 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,42827,1731212465525 2024-11-10T04:22:41,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:41,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:41,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:41,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:41,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:22:41,001 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:22:41,002 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:22:41,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:41,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:41,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:41,002 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,43425,1731212466265' ***** 2024-11-10T04:22:41,002 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:22:41,002 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:22:41,003 INFO [RS:0;d1d9e3766cb1:43425 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:22:41,003 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:22:41,003 INFO [RS:0;d1d9e3766cb1:43425 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:22:41,003 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(3091): Received CLOSE for d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:22:41,004 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,43425,1731212466265 2024-11-10T04:22:41,004 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:22:41,004 INFO [RS:0;d1d9e3766cb1:43425 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:43425. 2024-11-10T04:22:41,004 DEBUG [RS:0;d1d9e3766cb1:43425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:22:41,004 DEBUG [RS:0;d1d9e3766cb1:43425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:41,004 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:22:41,004 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:22:41,004 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:22:41,004 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d90b6241a0d0a0b09c9951c9f565effe, disabling compactions & flushes 2024-11-10T04:22:41,004 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:22:41,004 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:22:41,004 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:22:41,004 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. after waiting 0 ms 2024-11-10T04:22:41,004 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:22:41,005 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T04:22:41,005 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d90b6241a0d0a0b09c9951c9f565effe 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-10T04:22:41,005 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d90b6241a0d0a0b09c9951c9f565effe=TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.} 2024-11-10T04:22:41,005 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:22:41,005 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:22:41,005 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:22:41,005 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:22:41,005 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:22:41,005 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d90b6241a0d0a0b09c9951c9f565effe 2024-11-10T04:22:41,005 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-10T04:22:41,012 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/8461f2f7eab449cb8e5f85054987bfee is 1080, key is row0029/info:/1731212554983/Put/seqid=0 2024-11-10T04:22:41,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741850_1026 (size=8193) 2024-11-10T04:22:41,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741850_1026 (size=8193) 2024-11-10T04:22:41,020 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/8461f2f7eab449cb8e5f85054987bfee 2024-11-10T04:22:41,027 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/info/7cdcfbe039c54b4f8e4d36585b17171c is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe./info:regioninfo/1731212469339/Put/seqid=0 2024-11-10T04:22:41,030 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/.tmp/info/8461f2f7eab449cb8e5f85054987bfee as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/8461f2f7eab449cb8e5f85054987bfee 2024-11-10T04:22:41,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741851_1027 (size=7016) 2024-11-10T04:22:41,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741851_1027 (size=7016) 2024-11-10T04:22:41,035 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/info/7cdcfbe039c54b4f8e4d36585b17171c 2024-11-10T04:22:41,039 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/8461f2f7eab449cb8e5f85054987bfee, entries=3, sequenceid=48, filesize=8.0 K 2024-11-10T04:22:41,040 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for d90b6241a0d0a0b09c9951c9f565effe in 35ms, sequenceid=48, compaction requested=true 2024-11-10T04:22:41,041 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173, hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/e5555798a28b4deba417677fdd1b621a, hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/09d861f6daf649cb93d25fea195ed9fd] to archive 2024-11-10T04:22:41,045 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:22:41,048 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173 to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/117dbc11c66b4ed5acf741f183875173 2024-11-10T04:22:41,050 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/e5555798a28b4deba417677fdd1b621a to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/e5555798a28b4deba417677fdd1b621a 2024-11-10T04:22:41,052 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/09d861f6daf649cb93d25fea195ed9fd to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/info/09d861f6daf649cb93d25fea195ed9fd 2024-11-10T04:22:41,058 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/ns/3126d12a5f084882b382284f7787af81 is 43, key is default/ns:d/1731212468170/Put/seqid=0 2024-11-10T04:22:41,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741852_1028 (size=5153) 2024-11-10T04:22:41,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741852_1028 (size=5153) 2024-11-10T04:22:41,066 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/ns/3126d12a5f084882b382284f7787af81 2024-11-10T04:22:41,064 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d1d9e3766cb1:42827 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T04:22:41,069 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [117dbc11c66b4ed5acf741f183875173=12509, e5555798a28b4deba417677fdd1b621a=12509, 09d861f6daf649cb93d25fea195ed9fd=12509] 2024-11-10T04:22:41,075 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/default/TestLogRolling-testSlowSyncLogRolling/d90b6241a0d0a0b09c9951c9f565effe/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-11-10T04:22:41,078 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:22:41,078 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d90b6241a0d0a0b09c9951c9f565effe: Waiting for close lock at 1731212561004Running coprocessor pre-close hooks at 1731212561004Disabling compacts and flushes for region at 1731212561004Disabling writes for close at 1731212561004Obtaining lock to block concurrent updates at 1731212561005 (+1 ms)Preparing flush snapshotting stores in d90b6241a0d0a0b09c9951c9f565effe at 1731212561005Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731212561005Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. at 1731212561006 (+1 ms)Flushing d90b6241a0d0a0b09c9951c9f565effe/info: creating writer at 1731212561006Flushing d90b6241a0d0a0b09c9951c9f565effe/info: appending metadata at 1731212561011 (+5 ms)Flushing d90b6241a0d0a0b09c9951c9f565effe/info: closing flushed file at 1731212561011Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ed24f: reopening flushed file at 1731212561029 (+18 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for d90b6241a0d0a0b09c9951c9f565effe in 35ms, sequenceid=48, compaction requested=true at 1731212561040 (+11 ms)Writing region close event to WAL at 1731212561070 (+30 ms)Running coprocessor post-close hooks at 1731212561076 (+6 ms)Closed at 1731212561078 (+2 ms) 2024-11-10T04:22:41,079 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731212468482.d90b6241a0d0a0b09c9951c9f565effe. 2024-11-10T04:22:41,091 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/table/9a5e2c1b2e584fd0847eddbc8554f573 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731212469358/Put/seqid=0 2024-11-10T04:22:41,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741853_1029 (size=5396) 2024-11-10T04:22:41,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741853_1029 (size=5396) 2024-11-10T04:22:41,205 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:22:41,406 DEBUG [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:22:41,414 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T04:22:41,415 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T04:22:41,419 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:22:41,499 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/table/9a5e2c1b2e584fd0847eddbc8554f573 2024-11-10T04:22:41,507 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/info/7cdcfbe039c54b4f8e4d36585b17171c as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/info/7cdcfbe039c54b4f8e4d36585b17171c 2024-11-10T04:22:41,516 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/info/7cdcfbe039c54b4f8e4d36585b17171c, entries=10, sequenceid=11, filesize=6.9 K 2024-11-10T04:22:41,518 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/ns/3126d12a5f084882b382284f7787af81 as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/ns/3126d12a5f084882b382284f7787af81 2024-11-10T04:22:41,526 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/ns/3126d12a5f084882b382284f7787af81, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T04:22:41,528 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/.tmp/table/9a5e2c1b2e584fd0847eddbc8554f573 as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/table/9a5e2c1b2e584fd0847eddbc8554f573 2024-11-10T04:22:41,535 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/table/9a5e2c1b2e584fd0847eddbc8554f573, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T04:22:41,537 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 532ms, sequenceid=11, compaction requested=false 2024-11-10T04:22:41,543 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T04:22:41,544 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:22:41,544 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:22:41,544 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212561005Running coprocessor pre-close hooks at 1731212561005Disabling compacts and flushes for region at 1731212561005Disabling writes for close at 1731212561005Obtaining lock to block concurrent updates at 1731212561005Preparing flush snapshotting stores in 1588230740 at 1731212561005Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731212561006 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731212561007 (+1 ms)Flushing 1588230740/info: creating writer at 1731212561007Flushing 1588230740/info: appending metadata at 1731212561027 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731212561027Flushing 1588230740/ns: creating writer at 1731212561042 (+15 ms)Flushing 1588230740/ns: appending metadata at 1731212561058 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731212561058Flushing 1588230740/table: creating writer at 1731212561074 (+16 ms)Flushing 1588230740/table: appending metadata at 1731212561091 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731212561091Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5522a62c: reopening flushed file at 1731212561506 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fc4cca9: reopening flushed file at 1731212561516 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7954cdb1: reopening flushed file at 1731212561526 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 532ms, sequenceid=11, compaction requested=false at 1731212561537 (+11 ms)Writing region close event to WAL at 1731212561539 (+2 ms)Running coprocessor post-close hooks at 1731212561544 (+5 ms)Closed at 1731212561544 2024-11-10T04:22:41,544 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:22:41,606 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,43425,1731212466265; all regions closed. 2024-11-10T04:22:41,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741834_1010 (size=3066) 2024-11-10T04:22:41,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741834_1010 (size=3066) 2024-11-10T04:22:41,614 DEBUG [RS:0;d1d9e3766cb1:43425 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs 2024-11-10T04:22:41,614 INFO [RS:0;d1d9e3766cb1:43425 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C43425%2C1731212466265.meta:.meta(num 1731212468020) 2024-11-10T04:22:41,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741847_1023 (size=12695) 2024-11-10T04:22:41,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741847_1023 (size=12695) 2024-11-10T04:22:41,621 DEBUG [RS:0;d1d9e3766cb1:43425 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/oldWALs 2024-11-10T04:22:41,621 INFO [RS:0;d1d9e3766cb1:43425 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C43425%2C1731212466265:(num 1731212540956) 2024-11-10T04:22:41,621 DEBUG [RS:0;d1d9e3766cb1:43425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:41,621 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:22:41,621 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:22:41,621 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:22:41,621 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:22:41,622 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:22:41,622 INFO [RS:0;d1d9e3766cb1:43425 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43425 2024-11-10T04:22:41,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:22:41,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,43425,1731212466265 2024-11-10T04:22:41,626 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:22:41,628 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,43425,1731212466265] 2024-11-10T04:22:41,630 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,43425,1731212466265 already deleted, retry=false 2024-11-10T04:22:41,630 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,43425,1731212466265 expired; onlineServers=0 2024-11-10T04:22:41,630 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,42827,1731212465525' ***** 2024-11-10T04:22:41,630 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:22:41,630 INFO [M:0;d1d9e3766cb1:42827 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:22:41,630 INFO [M:0;d1d9e3766cb1:42827 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:22:41,630 DEBUG [M:0;d1d9e3766cb1:42827 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:22:41,631 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:22:41,631 DEBUG [M:0;d1d9e3766cb1:42827 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:22:41,631 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212467270 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212467270,5,FailOnTimeoutGroup] 2024-11-10T04:22:41,631 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212467272 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212467272,5,FailOnTimeoutGroup] 2024-11-10T04:22:41,631 INFO [M:0;d1d9e3766cb1:42827 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:22:41,631 INFO [M:0;d1d9e3766cb1:42827 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:22:41,631 DEBUG [M:0;d1d9e3766cb1:42827 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:22:41,631 INFO [M:0;d1d9e3766cb1:42827 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:22:41,631 INFO [M:0;d1d9e3766cb1:42827 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:22:41,632 INFO [M:0;d1d9e3766cb1:42827 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:22:41,632 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:22:41,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:22:41,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:41,633 DEBUG [M:0;d1d9e3766cb1:42827 {}] zookeeper.ZKUtil(347): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:22:41,633 WARN [M:0;d1d9e3766cb1:42827 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:22:41,634 INFO [M:0;d1d9e3766cb1:42827 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/.lastflushedseqids 2024-11-10T04:22:41,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741854_1030 (size=130) 2024-11-10T04:22:41,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741854_1030 (size=130) 2024-11-10T04:22:41,647 INFO [M:0;d1d9e3766cb1:42827 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:22:41,647 INFO [M:0;d1d9e3766cb1:42827 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:22:41,647 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:22:41,647 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:41,647 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:41,647 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:22:41,647 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:41,648 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB 2024-11-10T04:22:41,666 DEBUG [M:0;d1d9e3766cb1:42827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8945bfbecd984a499c96b06ff5dd4918 is 82, key is hbase:meta,,1/info:regioninfo/1731212468091/Put/seqid=0 2024-11-10T04:22:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741855_1031 (size=5672) 2024-11-10T04:22:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741855_1031 (size=5672) 2024-11-10T04:22:41,672 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8945bfbecd984a499c96b06ff5dd4918 2024-11-10T04:22:41,695 DEBUG [M:0;d1d9e3766cb1:42827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/efc3f1258a77486eb2fea0a9a79061e0 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731212469366/Put/seqid=0 2024-11-10T04:22:41,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741856_1032 (size=6247) 2024-11-10T04:22:41,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741856_1032 (size=6247) 2024-11-10T04:22:41,702 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/efc3f1258a77486eb2fea0a9a79061e0 2024-11-10T04:22:41,708 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for efc3f1258a77486eb2fea0a9a79061e0 2024-11-10T04:22:41,724 DEBUG [M:0;d1d9e3766cb1:42827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2079651a6e4f90b1aaf95a4320c69b is 69, key is d1d9e3766cb1,43425,1731212466265/rs:state/1731212467324/Put/seqid=0 2024-11-10T04:22:41,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:41,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43425-0x101906947960001, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:41,729 INFO [RS:0;d1d9e3766cb1:43425 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:22:41,729 INFO [RS:0;d1d9e3766cb1:43425 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,43425,1731212466265; zookeeper connection closed. 2024-11-10T04:22:41,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741857_1033 (size=5156) 2024-11-10T04:22:41,730 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e4fa109 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e4fa109 2024-11-10T04:22:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741857_1033 (size=5156) 2024-11-10T04:22:41,730 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:22:41,731 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2079651a6e4f90b1aaf95a4320c69b 2024-11-10T04:22:41,751 DEBUG [M:0;d1d9e3766cb1:42827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d768768c78c4bf58f71112d71b89f7d is 52, key is load_balancer_on/state:d/1731212468461/Put/seqid=0 2024-11-10T04:22:41,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741858_1034 (size=5056) 2024-11-10T04:22:41,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741858_1034 (size=5056) 2024-11-10T04:22:41,758 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d768768c78c4bf58f71112d71b89f7d 2024-11-10T04:22:41,765 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8945bfbecd984a499c96b06ff5dd4918 as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8945bfbecd984a499c96b06ff5dd4918 2024-11-10T04:22:41,772 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8945bfbecd984a499c96b06ff5dd4918, entries=8, sequenceid=59, filesize=5.5 K 2024-11-10T04:22:41,773 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/efc3f1258a77486eb2fea0a9a79061e0 as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/efc3f1258a77486eb2fea0a9a79061e0 2024-11-10T04:22:41,780 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for efc3f1258a77486eb2fea0a9a79061e0 2024-11-10T04:22:41,780 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/efc3f1258a77486eb2fea0a9a79061e0, entries=6, sequenceid=59, filesize=6.1 K 2024-11-10T04:22:41,781 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2079651a6e4f90b1aaf95a4320c69b as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b2079651a6e4f90b1aaf95a4320c69b 2024-11-10T04:22:41,787 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b2079651a6e4f90b1aaf95a4320c69b, entries=1, sequenceid=59, filesize=5.0 K 2024-11-10T04:22:41,788 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d768768c78c4bf58f71112d71b89f7d as hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9d768768c78c4bf58f71112d71b89f7d 2024-11-10T04:22:41,794 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9d768768c78c4bf58f71112d71b89f7d, entries=1, sequenceid=59, filesize=4.9 K 2024-11-10T04:22:41,796 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 148ms, sequenceid=59, compaction requested=false 2024-11-10T04:22:41,797 INFO [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:41,797 DEBUG [M:0;d1d9e3766cb1:42827 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212561647Disabling compacts and flushes for region at 1731212561647Disabling writes for close at 1731212561647Obtaining lock to block concurrent updates at 1731212561648 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212561648Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1731212561648Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212561649 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212561649Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212561665 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212561665Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212561679 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212561694 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212561694Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212561708 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212561723 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212561723Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731212561737 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731212561751 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731212561751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18718f0b: reopening flushed file at 1731212561764 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a60e4b3: reopening flushed file at 1731212561772 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd5a074: reopening flushed file at 1731212561780 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3144a0a6: reopening flushed file at 1731212561787 (+7 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 148ms, sequenceid=59, compaction requested=false at 1731212561796 (+9 ms)Writing region close event to WAL at 1731212561797 (+1 ms)Closed at 1731212561797 2024-11-10T04:22:41,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:41,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39761 is added to blk_1073741830_1006 (size=27973) 2024-11-10T04:22:41,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44523 is added to blk_1073741830_1006 (size=27973) 2024-11-10T04:22:41,802 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:22:41,802 INFO [M:0;d1d9e3766cb1:42827 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:22:41,802 INFO [M:0;d1d9e3766cb1:42827 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42827 2024-11-10T04:22:41,803 INFO [M:0;d1d9e3766cb1:42827 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:22:41,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:41,905 INFO [M:0;d1d9e3766cb1:42827 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:22:41,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42827-0x101906947960000, quorum=127.0.0.1:52254, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:41,910 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:41,912 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:41,912 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:41,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:41,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:41,915 WARN [BP-1998714310-172.17.0.2-1731212462530 heartbeating to localhost/127.0.0.1:45907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:22:41,915 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:22:41,915 WARN [BP-1998714310-172.17.0.2-1731212462530 heartbeating to localhost/127.0.0.1:45907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1998714310-172.17.0.2-1731212462530 (Datanode Uuid 8d606198-1d31-4f46-995e-fc52a703af32) service to localhost/127.0.0.1:45907 2024-11-10T04:22:41,915 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:22:41,917 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data3/current/BP-1998714310-172.17.0.2-1731212462530 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:41,917 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data4/current/BP-1998714310-172.17.0.2-1731212462530 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:41,917 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:22:41,925 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:41,925 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:41,925 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:41,925 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:41,926 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:41,927 WARN [BP-1998714310-172.17.0.2-1731212462530 heartbeating to localhost/127.0.0.1:45907 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:22:41,927 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:22:41,927 WARN [BP-1998714310-172.17.0.2-1731212462530 heartbeating to localhost/127.0.0.1:45907 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1998714310-172.17.0.2-1731212462530 (Datanode Uuid 35d6de79-48d2-4c36-94c1-86f9b18b9263) service to localhost/127.0.0.1:45907 2024-11-10T04:22:41,927 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:22:41,928 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data1/current/BP-1998714310-172.17.0.2-1731212462530 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:41,928 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/cluster_793c898f-ce3e-6d48-1670-138ed0922552/data/data2/current/BP-1998714310-172.17.0.2-1731212462530 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:41,928 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:22:41,940 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:22:41,941 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:41,941 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:41,941 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:41,941 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:41,949 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:22:41,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:22:41,988 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45907 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45907 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: regionserver/d1d9e3766cb1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@efc635d java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45907 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: master/d1d9e3766cb1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45907 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45907 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:45907 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45907 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45907 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/d1d9e3766cb1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=16 (was 17), ProcessCount=11 (was 11), AvailableMemoryMB=7623 (was 8120) 2024-11-10T04:22:41,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=16, ProcessCount=11, AvailableMemoryMB=7623 2024-11-10T04:22:41,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:22:41,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.log.dir so I do NOT create it in target/test-data/0772ab59-52db-74d9-1722-e4b173f15283 2024-11-10T04:22:41,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/468f958e-2964-af6f-9111-a294cbb2c126/hadoop.tmp.dir so I do NOT create it in target/test-data/0772ab59-52db-74d9-1722-e4b173f15283 2024-11-10T04:22:41,995 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8, deleteOnExit=true 2024-11-10T04:22:41,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:22:41,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/test.cache.data in system properties and HBase conf 2024-11-10T04:22:41,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:22:41,996 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:22:41,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:22:41,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:22:42,011 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:22:42,079 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:42,085 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:42,086 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:42,086 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:42,086 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:22:42,087 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:42,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ae773f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:42,088 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@630e1a46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:42,204 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d195093{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/java.io.tmpdir/jetty-localhost-39231-hadoop-hdfs-3_4_1-tests_jar-_-any-2227874589904467998/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:22:42,205 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@476d81{HTTP/1.1, (http/1.1)}{localhost:39231} 2024-11-10T04:22:42,205 INFO [Time-limited test {}] server.Server(415): Started @101750ms 2024-11-10T04:22:42,219 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:22:42,286 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:42,289 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:42,290 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:42,290 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:42,290 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:22:42,290 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@670e4080{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:42,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7247ee1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:42,405 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6a249094{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/java.io.tmpdir/jetty-localhost-36713-hadoop-hdfs-3_4_1-tests_jar-_-any-2287677151703111038/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:42,406 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@35445cbd{HTTP/1.1, (http/1.1)}{localhost:36713} 2024-11-10T04:22:42,406 INFO [Time-limited test {}] server.Server(415): Started @101951ms 2024-11-10T04:22:42,407 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:22:42,447 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:42,451 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:42,451 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:42,451 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:42,451 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:22:42,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18f27499{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:42,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64e5ce98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:42,504 WARN [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data1/current/BP-315219371-172.17.0.2-1731212562030/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:42,504 WARN [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data2/current/BP-315219371-172.17.0.2-1731212562030/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:42,523 WARN [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:22:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2358b7ee09272783 with lease ID 0x26784d4dc85e6a1b: Processing first storage report for DS-d39e4200-b05e-45d1-a4cb-21b793a46d1f from datanode DatanodeRegistration(127.0.0.1:37243, datanodeUuid=28940095-1bfb-4a17-aa9e-232c267e683f, infoPort=44539, infoSecurePort=0, ipcPort=41359, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030) 2024-11-10T04:22:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2358b7ee09272783 with lease ID 0x26784d4dc85e6a1b: from storage DS-d39e4200-b05e-45d1-a4cb-21b793a46d1f node DatanodeRegistration(127.0.0.1:37243, datanodeUuid=28940095-1bfb-4a17-aa9e-232c267e683f, infoPort=44539, infoSecurePort=0, ipcPort=41359, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2358b7ee09272783 with lease ID 0x26784d4dc85e6a1b: Processing first storage report for DS-8fbdeb20-391a-4aba-9f4c-db9c99d6adc2 from datanode DatanodeRegistration(127.0.0.1:37243, datanodeUuid=28940095-1bfb-4a17-aa9e-232c267e683f, infoPort=44539, infoSecurePort=0, ipcPort=41359, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030) 2024-11-10T04:22:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2358b7ee09272783 with lease ID 0x26784d4dc85e6a1b: from storage DS-8fbdeb20-391a-4aba-9f4c-db9c99d6adc2 node DatanodeRegistration(127.0.0.1:37243, datanodeUuid=28940095-1bfb-4a17-aa9e-232c267e683f, infoPort=44539, infoSecurePort=0, ipcPort=41359, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:42,571 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@254b7745{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/java.io.tmpdir/jetty-localhost-33363-hadoop-hdfs-3_4_1-tests_jar-_-any-16419106714933265566/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:42,571 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@86e294e{HTTP/1.1, (http/1.1)}{localhost:33363} 2024-11-10T04:22:42,571 INFO [Time-limited test {}] server.Server(415): Started @102116ms 2024-11-10T04:22:42,573 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:22:42,668 WARN [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data3/current/BP-315219371-172.17.0.2-1731212562030/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:42,669 WARN [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data4/current/BP-315219371-172.17.0.2-1731212562030/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:42,694 WARN [Thread-453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:22:42,696 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x315d6d24dd448fbd with lease ID 0x26784d4dc85e6a1c: Processing first storage report for DS-afe69f9f-134f-4e8d-a37a-00e873e1774e from datanode DatanodeRegistration(127.0.0.1:46093, datanodeUuid=c9a9ea3c-7ba0-410a-b7f8-361041df32c7, infoPort=41659, infoSecurePort=0, ipcPort=45597, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030) 2024-11-10T04:22:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x315d6d24dd448fbd with lease ID 0x26784d4dc85e6a1c: from storage DS-afe69f9f-134f-4e8d-a37a-00e873e1774e node DatanodeRegistration(127.0.0.1:46093, datanodeUuid=c9a9ea3c-7ba0-410a-b7f8-361041df32c7, infoPort=41659, infoSecurePort=0, ipcPort=45597, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x315d6d24dd448fbd with lease ID 0x26784d4dc85e6a1c: Processing first storage report for DS-7a346a4a-3818-4a5e-b161-e2441159a533 from datanode DatanodeRegistration(127.0.0.1:46093, datanodeUuid=c9a9ea3c-7ba0-410a-b7f8-361041df32c7, infoPort=41659, infoSecurePort=0, ipcPort=45597, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030) 2024-11-10T04:22:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x315d6d24dd448fbd with lease ID 0x26784d4dc85e6a1c: from storage DS-7a346a4a-3818-4a5e-b161-e2441159a533 node DatanodeRegistration(127.0.0.1:46093, datanodeUuid=c9a9ea3c-7ba0-410a-b7f8-361041df32c7, infoPort=41659, infoSecurePort=0, ipcPort=45597, storageInfo=lv=-57;cid=testClusterID;nsid=1382677080;c=1731212562030), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:42,702 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283 2024-11-10T04:22:42,705 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/zookeeper_0, clientPort=49989, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:22:42,706 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49989 2024-11-10T04:22:42,706 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:42,708 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:42,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:22:42,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:22:42,720 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b with version=8 2024-11-10T04:22:42,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase-staging 2024-11-10T04:22:42,722 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:22:42,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:42,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:42,722 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:22:42,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:42,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:22:42,723 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:22:42,723 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:22:42,723 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37411 2024-11-10T04:22:42,725 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37411 connecting to ZooKeeper ensemble=127.0.0.1:49989 2024-11-10T04:22:42,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:374110x0, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:22:42,732 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37411-0x101906ac6780000 connected 2024-11-10T04:22:42,748 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:42,749 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:42,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:42,752 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b, hbase.cluster.distributed=false 2024-11-10T04:22:42,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:22:42,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37411 2024-11-10T04:22:42,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37411 2024-11-10T04:22:42,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37411 2024-11-10T04:22:42,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37411 2024-11-10T04:22:42,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37411 2024-11-10T04:22:42,773 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:22:42,773 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:42,773 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:42,773 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:22:42,773 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:42,773 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:22:42,773 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:22:42,773 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:22:42,774 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43595 2024-11-10T04:22:42,775 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43595 connecting to ZooKeeper ensemble=127.0.0.1:49989 2024-11-10T04:22:42,776 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:42,778 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:42,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435950x0, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:22:42,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:435950x0, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:42,786 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43595-0x101906ac6780001 connected 2024-11-10T04:22:42,787 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:22:42,787 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:22:42,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:22:42,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:22:42,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43595 2024-11-10T04:22:42,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43595 2024-11-10T04:22:42,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43595 2024-11-10T04:22:42,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43595 2024-11-10T04:22:42,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43595 2024-11-10T04:22:42,809 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:37411 2024-11-10T04:22:42,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:42,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:42,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:42,812 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:42,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:22:42,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,815 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:22:42,815 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,37411,1731212562722 from backup master directory 2024-11-10T04:22:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:42,817 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:22:42,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:42,817 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:42,823 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/hbase.id] with ID: ef5e2320-cce5-450b-8eaf-1328a497349c 2024-11-10T04:22:42,823 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/.tmp/hbase.id 2024-11-10T04:22:42,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:22:42,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:22:42,832 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/.tmp/hbase.id]:[hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/hbase.id] 2024-11-10T04:22:42,847 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:42,847 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:22:42,849 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-10T04:22:42,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:22:42,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:22:42,862 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:22:42,863 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:22:42,864 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:22:42,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:22:42,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:22:42,873 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store 2024-11-10T04:22:42,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:22:42,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:22:42,881 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:42,882 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:22:42,882 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:42,882 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:42,882 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:22:42,882 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:42,882 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:42,882 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212562882Disabling compacts and flushes for region at 1731212562882Disabling writes for close at 1731212562882Writing region close event to WAL at 1731212562882Closed at 1731212562882 2024-11-10T04:22:42,884 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/.initializing 2024-11-10T04:22:42,884 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/WALs/d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:42,887 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C37411%2C1731212562722, suffix=, logDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/WALs/d1d9e3766cb1,37411,1731212562722, archiveDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/oldWALs, maxLogs=10 2024-11-10T04:22:42,888 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C37411%2C1731212562722.1731212562888 2024-11-10T04:22:42,894 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/WALs/d1d9e3766cb1,37411,1731212562722/d1d9e3766cb1%2C37411%2C1731212562722.1731212562888 2024-11-10T04:22:42,894 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41659:41659),(127.0.0.1/127.0.0.1:44539:44539)] 2024-11-10T04:22:42,895 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:22:42,895 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:42,895 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,895 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,897 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:22:42,899 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:42,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,901 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:22:42,901 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,902 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:22:42,902 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,904 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:22:42,904 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,905 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:22:42,905 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,906 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:22:42,906 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:22:42,907 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,908 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,908 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,910 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,910 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,911 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:22:42,912 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:42,915 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:22:42,915 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=724730, jitterRate=-0.0784587562084198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:22:42,917 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212562895Initializing all the Stores at 1731212562897 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212562897Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212562897Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212562897Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212562897Cleaning up temporary data from old regions at 1731212562910 (+13 ms)Region opened successfully at 1731212562916 (+6 ms) 2024-11-10T04:22:42,917 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:22:42,921 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acb6ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:22:42,921 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:22:42,922 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:22:42,922 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:22:42,922 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:22:42,922 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T04:22:42,923 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T04:22:42,923 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:22:42,925 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:22:42,926 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:22:42,927 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:22:42,928 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:22:42,929 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:22:42,930 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:22:42,931 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:22:42,931 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:22:42,933 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:22:42,934 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:22:42,935 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:22:42,937 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:22:42,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:22:42,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:42,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:42,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,37411,1731212562722, sessionid=0x101906ac6780000, setting cluster-up flag (Was=false) 2024-11-10T04:22:42,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,951 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:22:42,952 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:42,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:42,962 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:22:42,963 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:42,964 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:22:42,966 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:42,966 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:22:42,966 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:22:42,967 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,37411,1731212562722 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:22:42,968 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:42,968 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:42,968 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:42,968 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:42,969 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:22:42,969 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:42,969 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:22:42,969 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:42,971 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:42,971 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:22:42,972 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,972 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212592972 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:22:42,972 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:42,973 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:22:42,974 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:22:42,974 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:22:42,974 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:22:42,974 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:22:42,975 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212562974,5,FailOnTimeoutGroup] 2024-11-10T04:22:42,975 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212562975,5,FailOnTimeoutGroup] 2024-11-10T04:22:42,975 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:42,975 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:22:42,975 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:42,975 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:42,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:22:42,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:22:42,981 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:22:42,981 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b 2024-11-10T04:22:42,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:22:42,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:22:42,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:42,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:22:42,993 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:22:42,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:42,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:22:42,995 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:22:42,995 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:42,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:22:42,997 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:22:42,997 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:42,998 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:42,998 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:22:42,998 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(746): ClusterId : ef5e2320-cce5-450b-8eaf-1328a497349c 2024-11-10T04:22:42,998 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:22:42,999 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:22:42,999 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:43,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:43,000 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:22:43,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740 2024-11-10T04:22:43,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740 2024-11-10T04:22:43,001 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:22:43,001 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:22:43,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:22:43,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:22:43,004 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:22:43,004 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:22:43,004 DEBUG [RS:0;d1d9e3766cb1:43595 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0a0b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:22:43,005 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:22:43,009 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:22:43,009 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=760415, jitterRate=-0.03308340907096863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:22:43,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212562989Initializing all the Stores at 1731212562990 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212562991 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212562991Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212562991Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212562991Cleaning up temporary data from old regions at 1731212563003 (+12 ms)Region opened successfully at 1731212563010 (+7 ms) 2024-11-10T04:22:43,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:22:43,011 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:22:43,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:22:43,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:22:43,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:22:43,011 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:22:43,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212563011Disabling compacts and flushes for region at 1731212563011Disabling writes for close at 1731212563011Writing region close event to WAL at 1731212563011Closed at 1731212563011 2024-11-10T04:22:43,013 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:43,013 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:22:43,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:22:43,015 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:22:43,017 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:22:43,019 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:43595 2024-11-10T04:22:43,019 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:22:43,019 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:22:43,019 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:22:43,020 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,37411,1731212562722 with port=43595, startcode=1731212562772 2024-11-10T04:22:43,020 DEBUG [RS:0;d1d9e3766cb1:43595 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:22:43,023 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39749, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:22:43,024 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37411 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,024 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37411 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,026 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b 2024-11-10T04:22:43,026 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38185 2024-11-10T04:22:43,026 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:22:43,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:22:43,029 DEBUG [RS:0;d1d9e3766cb1:43595 {}] zookeeper.ZKUtil(111): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,029 WARN [RS:0;d1d9e3766cb1:43595 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:22:43,029 INFO [RS:0;d1d9e3766cb1:43595 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:22:43,029 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,43595,1731212562772] 2024-11-10T04:22:43,029 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/WALs/d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,033 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:22:43,036 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:22:43,036 INFO [RS:0;d1d9e3766cb1:43595 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:22:43,036 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,040 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:22:43,041 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:22:43,041 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,041 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,041 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,041 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,041 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,041 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,041 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:22:43,041 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,042 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,042 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,042 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,042 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,042 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:43,042 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:22:43,042 DEBUG [RS:0;d1d9e3766cb1:43595 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:22:43,042 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,042 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,042 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,042 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,043 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,043 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43595,1731212562772-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:22:43,059 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:22:43,059 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43595,1731212562772-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,059 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,059 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.Replication(171): d1d9e3766cb1,43595,1731212562772 started 2024-11-10T04:22:43,074 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,075 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,43595,1731212562772, RpcServer on d1d9e3766cb1/172.17.0.2:43595, sessionid=0x101906ac6780001 2024-11-10T04:22:43,075 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:22:43,075 DEBUG [RS:0;d1d9e3766cb1:43595 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,075 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,43595,1731212562772' 2024-11-10T04:22:43,075 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:22:43,075 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:22:43,076 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:22:43,076 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:22:43,076 DEBUG [RS:0;d1d9e3766cb1:43595 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,076 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,43595,1731212562772' 2024-11-10T04:22:43,076 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:22:43,077 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:22:43,077 DEBUG [RS:0;d1d9e3766cb1:43595 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:22:43,077 INFO [RS:0;d1d9e3766cb1:43595 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:22:43,077 INFO [RS:0;d1d9e3766cb1:43595 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:22:43,167 WARN [d1d9e3766cb1:37411 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:22:43,180 INFO [RS:0;d1d9e3766cb1:43595 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C43595%2C1731212562772, suffix=, logDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/WALs/d1d9e3766cb1,43595,1731212562772, archiveDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/oldWALs, maxLogs=32 2024-11-10T04:22:43,182 INFO [RS:0;d1d9e3766cb1:43595 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43595%2C1731212562772.1731212563182 2024-11-10T04:22:43,189 INFO [RS:0;d1d9e3766cb1:43595 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/WALs/d1d9e3766cb1,43595,1731212562772/d1d9e3766cb1%2C43595%2C1731212562772.1731212563182 2024-11-10T04:22:43,190 DEBUG [RS:0;d1d9e3766cb1:43595 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41659:41659),(127.0.0.1/127.0.0.1:44539:44539)] 2024-11-10T04:22:43,418 DEBUG [d1d9e3766cb1:37411 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:22:43,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,420 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,43595,1731212562772, state=OPENING 2024-11-10T04:22:43,422 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:22:43,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:43,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:43,425 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:22:43,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:43,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,43595,1731212562772}] 2024-11-10T04:22:43,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:43,579 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:22:43,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51709, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:22:43,585 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:22:43,586 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:22:43,588 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C43595%2C1731212562772.meta, suffix=.meta, logDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/WALs/d1d9e3766cb1,43595,1731212562772, archiveDir=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/oldWALs, maxLogs=32 2024-11-10T04:22:43,590 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43595%2C1731212562772.meta.1731212563589.meta 2024-11-10T04:22:43,596 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/WALs/d1d9e3766cb1,43595,1731212562772/d1d9e3766cb1%2C43595%2C1731212562772.meta.1731212563589.meta 2024-11-10T04:22:43,597 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44539:44539),(127.0.0.1/127.0.0.1:41659:41659)] 2024-11-10T04:22:43,598 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:22:43,598 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:22:43,598 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:22:43,599 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:22:43,599 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:22:43,599 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:43,599 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:22:43,599 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:22:43,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:22:43,602 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:22:43,602 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:43,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:43,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:22:43,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:22:43,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:43,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:43,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:22:43,606 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:22:43,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:43,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:43,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:22:43,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:22:43,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:43,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:43,608 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:22:43,609 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740 2024-11-10T04:22:43,610 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740 2024-11-10T04:22:43,611 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:22:43,612 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:22:43,612 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:22:43,614 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:22:43,615 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=699572, jitterRate=-0.1104489266872406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:22:43,615 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:22:43,616 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212563600Writing region info on filesystem at 1731212563600Initializing all the Stores at 1731212563601 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212563601Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212563601Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212563601Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212563601Cleaning up temporary data from old regions at 1731212563612 (+11 ms)Running coprocessor post-open hooks at 1731212563615 (+3 ms)Region opened successfully at 1731212563616 (+1 ms) 2024-11-10T04:22:43,617 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212563578 2024-11-10T04:22:43,620 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:22:43,620 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:22:43,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,623 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,43595,1731212562772, state=OPEN 2024-11-10T04:22:43,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:22:43,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:22:43,628 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:43,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:43,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:22:43,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,43595,1731212562772 in 203 msec 2024-11-10T04:22:43,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:22:43,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 618 msec 2024-11-10T04:22:43,636 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:43,636 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:22:43,637 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:22:43,637 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,43595,1731212562772, seqNum=-1] 2024-11-10T04:22:43,638 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:22:43,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52167, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:22:43,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 680 msec 2024-11-10T04:22:43,647 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212563647, completionTime=-1 2024-11-10T04:22:43,647 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:22:43,647 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:22:43,649 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:22:43,649 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212623649 2024-11-10T04:22:43,649 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731212683649 2024-11-10T04:22:43,649 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T04:22:43,649 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,37411,1731212562722-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,650 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,37411,1731212562722-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,650 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,37411,1731212562722-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,650 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:37411, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,650 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,651 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,652 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:22:43,656 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.839sec 2024-11-10T04:22:43,656 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:22:43,656 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:22:43,656 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:22:43,656 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:22:43,656 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:22:43,656 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,37411,1731212562722-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:22:43,657 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,37411,1731212562722-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:22:43,660 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:22:43,660 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:22:43,660 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,37411,1731212562722-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:43,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffb657f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:22:43,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,37411,-1 for getting cluster id 2024-11-10T04:22:43,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:22:43,700 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ef5e2320-cce5-450b-8eaf-1328a497349c' 2024-11-10T04:22:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:22:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ef5e2320-cce5-450b-8eaf-1328a497349c" 2024-11-10T04:22:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72aebd01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:22:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,37411,-1] 2024-11-10T04:22:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:22:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:43,703 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:22:43,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5d9cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:22:43,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:22:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,43595,1731212562772, seqNum=-1] 2024-11-10T04:22:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:22:43,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:22:43,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:43,710 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:43,713 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T04:22:43,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:22:43,713 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:22:43,713 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:22:43,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:43,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:43,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:22:43,714 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:22:43,714 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=292436729, stopped=false 2024-11-10T04:22:43,714 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,37411,1731212562722 2024-11-10T04:22:43,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:43,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:43,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:43,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:43,717 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:22:43,717 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:22:43,717 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:22:43,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:43,717 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,43595,1731212562772' ***** 2024-11-10T04:22:43,717 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:22:43,717 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:43,717 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:43,718 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:22:43,718 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:22:43,718 INFO [RS:0;d1d9e3766cb1:43595 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:22:43,718 INFO [RS:0;d1d9e3766cb1:43595 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:22:43,718 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,718 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:22:43,718 INFO [RS:0;d1d9e3766cb1:43595 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:43595. 2024-11-10T04:22:43,718 DEBUG [RS:0;d1d9e3766cb1:43595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:22:43,718 DEBUG [RS:0;d1d9e3766cb1:43595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:43,719 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:22:43,719 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:22:43,719 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:22:43,719 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:22:43,719 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T04:22:43,719 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T04:22:43,719 DEBUG [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:22:43,719 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:22:43,719 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:22:43,719 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:22:43,719 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:22:43,719 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:22:43,720 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-10T04:22:43,738 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740/.tmp/ns/2a66facad3a64d7497a8c213306169ea is 43, key is default/ns:d/1731212563640/Put/seqid=0 2024-11-10T04:22:43,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741835_1011 (size=5153) 2024-11-10T04:22:43,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741835_1011 (size=5153) 2024-11-10T04:22:43,745 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740/.tmp/ns/2a66facad3a64d7497a8c213306169ea 2024-11-10T04:22:43,752 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740/.tmp/ns/2a66facad3a64d7497a8c213306169ea as hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740/ns/2a66facad3a64d7497a8c213306169ea 2024-11-10T04:22:43,759 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740/ns/2a66facad3a64d7497a8c213306169ea, entries=2, sequenceid=6, filesize=5.0 K 2024-11-10T04:22:43,760 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 40ms, sequenceid=6, compaction requested=false 2024-11-10T04:22:43,760 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:22:43,766 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T04:22:43,767 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:22:43,767 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:22:43,767 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212563719Running coprocessor pre-close hooks at 1731212563719Disabling compacts and flushes for region at 1731212563719Disabling writes for close at 1731212563719Obtaining lock to block concurrent updates at 1731212563720 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731212563720Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731212563720Flushing stores of hbase:meta,,1.1588230740 at 1731212563721 (+1 ms)Flushing 1588230740/ns: creating writer at 1731212563721Flushing 1588230740/ns: appending metadata at 1731212563738 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731212563738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e2e894e: reopening flushed file at 1731212563751 (+13 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 40ms, sequenceid=6, compaction requested=false at 1731212563760 (+9 ms)Writing region close event to WAL at 1731212563762 (+2 ms)Running coprocessor post-close hooks at 1731212563767 (+5 ms)Closed at 1731212563767 2024-11-10T04:22:43,767 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:22:43,919 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,43595,1731212562772; all regions closed. 2024-11-10T04:22:43,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741834_1010 (size=1152) 2024-11-10T04:22:43,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741834_1010 (size=1152) 2024-11-10T04:22:43,926 DEBUG [RS:0;d1d9e3766cb1:43595 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/oldWALs 2024-11-10T04:22:43,926 INFO [RS:0;d1d9e3766cb1:43595 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C43595%2C1731212562772.meta:.meta(num 1731212563589) 2024-11-10T04:22:43,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:43,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741833_1009 (size=93) 2024-11-10T04:22:43,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741833_1009 (size=93) 2024-11-10T04:22:43,931 DEBUG [RS:0;d1d9e3766cb1:43595 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/oldWALs 2024-11-10T04:22:43,931 INFO [RS:0;d1d9e3766cb1:43595 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C43595%2C1731212562772:(num 1731212563182) 2024-11-10T04:22:43,931 DEBUG [RS:0;d1d9e3766cb1:43595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:43,932 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:22:43,932 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:22:43,932 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T04:22:43,932 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:22:43,932 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:22:43,932 INFO [RS:0;d1d9e3766cb1:43595 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43595 2024-11-10T04:22:43,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,43595,1731212562772 2024-11-10T04:22:43,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:22:43,935 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:22:43,936 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,43595,1731212562772] 2024-11-10T04:22:43,938 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,43595,1731212562772 already deleted, retry=false 2024-11-10T04:22:43,938 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,43595,1731212562772 expired; onlineServers=0 2024-11-10T04:22:43,938 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,37411,1731212562722' ***** 2024-11-10T04:22:43,938 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:22:43,938 INFO [M:0;d1d9e3766cb1:37411 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:22:43,938 INFO [M:0;d1d9e3766cb1:37411 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:22:43,938 DEBUG [M:0;d1d9e3766cb1:37411 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:22:43,938 DEBUG [M:0;d1d9e3766cb1:37411 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:22:43,938 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:22:43,938 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212562975 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212562975,5,FailOnTimeoutGroup] 2024-11-10T04:22:43,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212562974 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212562974,5,FailOnTimeoutGroup] 2024-11-10T04:22:43,939 INFO [M:0;d1d9e3766cb1:37411 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:22:43,939 INFO [M:0;d1d9e3766cb1:37411 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:22:43,939 DEBUG [M:0;d1d9e3766cb1:37411 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:22:43,939 INFO [M:0;d1d9e3766cb1:37411 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:22:43,939 INFO [M:0;d1d9e3766cb1:37411 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:22:43,939 INFO [M:0;d1d9e3766cb1:37411 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:22:43,939 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:22:43,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:22:43,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:43,941 DEBUG [M:0;d1d9e3766cb1:37411 {}] zookeeper.ZKUtil(347): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:22:43,941 WARN [M:0;d1d9e3766cb1:37411 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:22:43,942 INFO [M:0;d1d9e3766cb1:37411 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/.lastflushedseqids 2024-11-10T04:22:43,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741836_1012 (size=99) 2024-11-10T04:22:43,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741836_1012 (size=99) 2024-11-10T04:22:43,948 INFO [M:0;d1d9e3766cb1:37411 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:22:43,948 INFO [M:0;d1d9e3766cb1:37411 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:22:43,949 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:22:43,949 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:43,949 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:43,949 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:22:43,949 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:43,949 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-10T04:22:43,967 DEBUG [M:0;d1d9e3766cb1:37411 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5e5acd70cc6f45869f11947275e1fbf9 is 82, key is hbase:meta,,1/info:regioninfo/1731212563621/Put/seqid=0 2024-11-10T04:22:43,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741837_1013 (size=5672) 2024-11-10T04:22:43,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741837_1013 (size=5672) 2024-11-10T04:22:43,973 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5e5acd70cc6f45869f11947275e1fbf9 2024-11-10T04:22:43,996 DEBUG [M:0;d1d9e3766cb1:37411 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8598422c51414731972db1cc29b9f86e is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731212563646/Put/seqid=0 2024-11-10T04:22:44,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741838_1014 (size=5275) 2024-11-10T04:22:44,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741838_1014 (size=5275) 2024-11-10T04:22:44,002 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8598422c51414731972db1cc29b9f86e 2024-11-10T04:22:44,024 DEBUG [M:0;d1d9e3766cb1:37411 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd55bf9b324c4883951bd435324f58c9 is 69, key is d1d9e3766cb1,43595,1731212562772/rs:state/1731212563024/Put/seqid=0 2024-11-10T04:22:44,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741839_1015 (size=5156) 2024-11-10T04:22:44,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741839_1015 (size=5156) 2024-11-10T04:22:44,030 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd55bf9b324c4883951bd435324f58c9 2024-11-10T04:22:44,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:44,037 INFO [RS:0;d1d9e3766cb1:43595 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:22:44,037 INFO [RS:0;d1d9e3766cb1:43595 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,43595,1731212562772; zookeeper connection closed. 2024-11-10T04:22:44,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43595-0x101906ac6780001, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:44,037 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ca141 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ca141 2024-11-10T04:22:44,037 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:22:44,052 DEBUG [M:0;d1d9e3766cb1:37411 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fb701c28045148fa8588ad39e4a1b29c is 52, key is load_balancer_on/state:d/1731212563712/Put/seqid=0 2024-11-10T04:22:44,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741840_1016 (size=5056) 2024-11-10T04:22:44,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741840_1016 (size=5056) 2024-11-10T04:22:44,059 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fb701c28045148fa8588ad39e4a1b29c 2024-11-10T04:22:44,066 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5e5acd70cc6f45869f11947275e1fbf9 as hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5e5acd70cc6f45869f11947275e1fbf9 2024-11-10T04:22:44,072 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5e5acd70cc6f45869f11947275e1fbf9, entries=8, sequenceid=29, filesize=5.5 K 2024-11-10T04:22:44,073 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8598422c51414731972db1cc29b9f86e as hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8598422c51414731972db1cc29b9f86e 2024-11-10T04:22:44,078 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8598422c51414731972db1cc29b9f86e, entries=3, sequenceid=29, filesize=5.2 K 2024-11-10T04:22:44,079 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd55bf9b324c4883951bd435324f58c9 as hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd55bf9b324c4883951bd435324f58c9 2024-11-10T04:22:44,085 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd55bf9b324c4883951bd435324f58c9, entries=1, sequenceid=29, filesize=5.0 K 2024-11-10T04:22:44,086 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fb701c28045148fa8588ad39e4a1b29c as hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fb701c28045148fa8588ad39e4a1b29c 2024-11-10T04:22:44,091 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38185/user/jenkins/test-data/f3790e30-3b30-a907-e794-3c07ae9c049b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fb701c28045148fa8588ad39e4a1b29c, entries=1, sequenceid=29, filesize=4.9 K 2024-11-10T04:22:44,092 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 143ms, sequenceid=29, compaction requested=false 2024-11-10T04:22:44,094 INFO [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:44,094 DEBUG [M:0;d1d9e3766cb1:37411 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212563949Disabling compacts and flushes for region at 1731212563949Disabling writes for close at 1731212563949Obtaining lock to block concurrent updates at 1731212563949Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212563949Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731212563949Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212563950 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212563950Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212563966 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212563967 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212563980 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212563995 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212563995Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212564008 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212564024 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212564024Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731212564036 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731212564051 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731212564051Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b05e4e: reopening flushed file at 1731212564065 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24e92aeb: reopening flushed file at 1731212564072 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5240d312: reopening flushed file at 1731212564078 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b5e4956: reopening flushed file at 1731212564085 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 143ms, sequenceid=29, compaction requested=false at 1731212564092 (+7 ms)Writing region close event to WAL at 1731212564094 (+2 ms)Closed at 1731212564094 2024-11-10T04:22:44,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:44,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:44,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:44,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:44,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:44,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741830_1006 (size=10311) 2024-11-10T04:22:44,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741830_1006 (size=10311) 2024-11-10T04:22:44,098 INFO [M:0;d1d9e3766cb1:37411 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:22:44,098 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:22:44,098 INFO [M:0;d1d9e3766cb1:37411 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37411 2024-11-10T04:22:44,098 INFO [M:0;d1d9e3766cb1:37411 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:22:44,200 INFO [M:0;d1d9e3766cb1:37411 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:22:44,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:44,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37411-0x101906ac6780000, quorum=127.0.0.1:49989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:22:44,203 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@254b7745{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:44,203 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@86e294e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:44,203 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:44,203 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64e5ce98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:44,204 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18f27499{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:44,205 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:22:44,205 WARN [BP-315219371-172.17.0.2-1731212562030 heartbeating to localhost/127.0.0.1:38185 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:22:44,205 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:22:44,205 WARN [BP-315219371-172.17.0.2-1731212562030 heartbeating to localhost/127.0.0.1:38185 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-315219371-172.17.0.2-1731212562030 (Datanode Uuid c9a9ea3c-7ba0-410a-b7f8-361041df32c7) service to localhost/127.0.0.1:38185 2024-11-10T04:22:44,206 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data3/current/BP-315219371-172.17.0.2-1731212562030 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:44,206 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data4/current/BP-315219371-172.17.0.2-1731212562030 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:44,206 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:22:44,208 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6a249094{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:44,208 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@35445cbd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:44,209 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:44,209 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7247ee1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:44,209 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@670e4080{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:44,210 WARN [BP-315219371-172.17.0.2-1731212562030 heartbeating to localhost/127.0.0.1:38185 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:22:44,210 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:22:44,210 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:22:44,210 WARN [BP-315219371-172.17.0.2-1731212562030 heartbeating to localhost/127.0.0.1:38185 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-315219371-172.17.0.2-1731212562030 (Datanode Uuid 28940095-1bfb-4a17-aa9e-232c267e683f) service to localhost/127.0.0.1:38185 2024-11-10T04:22:44,211 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data1/current/BP-315219371-172.17.0.2-1731212562030 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:44,211 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/cluster_a1c35740-40ef-1aba-e13c-59beacafe1d8/data/data2/current/BP-315219371-172.17.0.2-1731212562030 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:44,212 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:22:44,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d195093{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:22:44,218 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@476d81{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:44,218 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:44,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@630e1a46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:44,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ae773f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:44,225 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:22:44,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:22:44,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:22:44,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.log.dir so I do NOT create it in target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc 2024-11-10T04:22:44,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0772ab59-52db-74d9-1722-e4b173f15283/hadoop.tmp.dir so I do NOT create it in target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4, deleteOnExit=true 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/test.cache.data in system properties and HBase conf 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:22:44,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:22:44,241 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:22:44,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:22:44,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:22:44,257 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:22:44,332 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:44,338 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:44,340 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:44,340 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:44,340 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:22:44,342 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:44,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a488aac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:44,344 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ed3a961{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:44,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cd2a640{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir/jetty-localhost-40613-hadoop-hdfs-3_4_1-tests_jar-_-any-13789166532920969704/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:22:44,465 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64b7b556{HTTP/1.1, (http/1.1)}{localhost:40613} 2024-11-10T04:22:44,465 INFO [Time-limited test {}] server.Server(415): Started @104010ms 2024-11-10T04:22:44,479 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:22:44,550 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:44,554 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:44,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:44,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:44,554 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:22:44,555 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@257cf4bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:44,555 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30e7c448{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:44,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@ab5393f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir/jetty-localhost-41657-hadoop-hdfs-3_4_1-tests_jar-_-any-6444042666758905886/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:44,670 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47f82e76{HTTP/1.1, (http/1.1)}{localhost:41657} 2024-11-10T04:22:44,670 INFO [Time-limited test {}] server.Server(415): Started @104216ms 2024-11-10T04:22:44,672 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:22:44,705 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:44,708 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:44,709 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:44,709 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:44,709 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:22:44,710 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69bbaec1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:44,710 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7720beab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:44,770 WARN [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data2/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:44,770 WARN [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data1/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:44,793 WARN [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:22:44,797 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6bd0d4fd83e353d5 with lease ID 0x88ed51acd8d0e3a0: Processing first storage report for DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4 from datanode DatanodeRegistration(127.0.0.1:44537, datanodeUuid=cc08ad11-27db-4910-a156-7a697eb22d59, infoPort=35575, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:44,797 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6bd0d4fd83e353d5 with lease ID 0x88ed51acd8d0e3a0: from storage DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4 node DatanodeRegistration(127.0.0.1:44537, datanodeUuid=cc08ad11-27db-4910-a156-7a697eb22d59, infoPort=35575, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:44,797 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6bd0d4fd83e353d5 with lease ID 0x88ed51acd8d0e3a0: Processing first storage report for DS-a19c85ae-d1a8-455a-838e-92c3cc081633 from datanode DatanodeRegistration(127.0.0.1:44537, datanodeUuid=cc08ad11-27db-4910-a156-7a697eb22d59, infoPort=35575, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:44,797 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6bd0d4fd83e353d5 with lease ID 0x88ed51acd8d0e3a0: from storage DS-a19c85ae-d1a8-455a-838e-92c3cc081633 node DatanodeRegistration(127.0.0.1:44537, datanodeUuid=cc08ad11-27db-4910-a156-7a697eb22d59, infoPort=35575, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:44,827 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21d5e4af{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir/jetty-localhost-40783-hadoop-hdfs-3_4_1-tests_jar-_-any-17228812024770883978/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:44,828 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@145c0180{HTTP/1.1, (http/1.1)}{localhost:40783} 2024-11-10T04:22:44,828 INFO [Time-limited test {}] server.Server(415): Started @104373ms 2024-11-10T04:22:44,829 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:22:44,927 WARN [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:44,927 WARN [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:44,950 WARN [Thread-672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:22:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf8dd5e952d3b0ac6 with lease ID 0x88ed51acd8d0e3a1: Processing first storage report for DS-2b0ba88d-61e6-4117-8f04-1db5508cb920 from datanode DatanodeRegistration(127.0.0.1:41831, datanodeUuid=36bda998-80e7-46b9-9d7b-4fc68015f300, infoPort=36337, infoSecurePort=0, ipcPort=34963, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf8dd5e952d3b0ac6 with lease ID 0x88ed51acd8d0e3a1: from storage DS-2b0ba88d-61e6-4117-8f04-1db5508cb920 node DatanodeRegistration(127.0.0.1:41831, datanodeUuid=36bda998-80e7-46b9-9d7b-4fc68015f300, infoPort=36337, infoSecurePort=0, ipcPort=34963, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf8dd5e952d3b0ac6 with lease ID 0x88ed51acd8d0e3a1: Processing first storage report for DS-a4cf91ae-f721-45d3-b122-1b6158c324d0 from datanode DatanodeRegistration(127.0.0.1:41831, datanodeUuid=36bda998-80e7-46b9-9d7b-4fc68015f300, infoPort=36337, infoSecurePort=0, ipcPort=34963, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf8dd5e952d3b0ac6 with lease ID 0x88ed51acd8d0e3a1: from storage DS-a4cf91ae-f721-45d3-b122-1b6158c324d0 node DatanodeRegistration(127.0.0.1:41831, datanodeUuid=36bda998-80e7-46b9-9d7b-4fc68015f300, infoPort=36337, infoSecurePort=0, ipcPort=34963, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:44,957 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc 2024-11-10T04:22:44,960 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/zookeeper_0, clientPort=60824, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:22:44,961 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60824 2024-11-10T04:22:44,961 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:44,963 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:22:44,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:22:44,974 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6 with version=8 2024-11-10T04:22:44,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase-staging 2024-11-10T04:22:44,976 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:22:44,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:44,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:44,976 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:22:44,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:44,977 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:22:44,977 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:22:44,977 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:22:44,977 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40031 2024-11-10T04:22:44,979 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40031 connecting to ZooKeeper ensemble=127.0.0.1:60824 2024-11-10T04:22:44,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400310x0, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:22:44,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40031-0x101906acf480000 connected 2024-11-10T04:22:45,008 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:45,010 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:45,012 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:45,012 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6, hbase.cluster.distributed=false 2024-11-10T04:22:45,014 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:22:45,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40031 2024-11-10T04:22:45,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40031 2024-11-10T04:22:45,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40031 2024-11-10T04:22:45,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40031 2024-11-10T04:22:45,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40031 2024-11-10T04:22:45,036 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:22:45,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:45,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:45,036 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:22:45,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:45,037 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:22:45,037 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:22:45,037 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:22:45,037 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38007 2024-11-10T04:22:45,039 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38007 connecting to ZooKeeper ensemble=127.0.0.1:60824 2024-11-10T04:22:45,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:45,041 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:45,043 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:22:45,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380070x0, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:22:45,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380070x0, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:22:45,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38007-0x101906acf480001 connected 2024-11-10T04:22:45,047 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:22:45,048 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:22:45,048 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:22:45,050 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:22:45,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38007 2024-11-10T04:22:45,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38007 2024-11-10T04:22:45,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38007 2024-11-10T04:22:45,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38007 2024-11-10T04:22:45,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38007 2024-11-10T04:22:45,067 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:40031 2024-11-10T04:22:45,067 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:45,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:45,071 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:22:45,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,073 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:22:45,074 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,40031,1731212564976 from backup master directory 2024-11-10T04:22:45,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:45,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:22:45,076 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:22:45,076 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,085 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/hbase.id] with ID: 7ba64496-0fa9-4701-bcde-2c2416623d24 2024-11-10T04:22:45,085 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/.tmp/hbase.id 2024-11-10T04:22:45,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:22:45,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:22:45,093 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/.tmp/hbase.id]:[hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/hbase.id] 2024-11-10T04:22:45,108 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:45,108 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:22:45,109 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T04:22:45,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:22:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:22:45,122 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:22:45,123 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:22:45,124 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:22:45,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:22:45,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:22:45,133 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store 2024-11-10T04:22:45,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:22:45,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:22:45,141 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:45,141 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:22:45,141 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:45,141 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:45,141 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:22:45,141 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:45,142 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:22:45,142 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212565141Disabling compacts and flushes for region at 1731212565141Disabling writes for close at 1731212565141Writing region close event to WAL at 1731212565142 (+1 ms)Closed at 1731212565142 2024-11-10T04:22:45,142 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/.initializing 2024-11-10T04:22:45,143 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,146 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C40031%2C1731212564976, suffix=, logDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976, archiveDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/oldWALs, maxLogs=10 2024-11-10T04:22:45,146 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 2024-11-10T04:22:45,152 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 2024-11-10T04:22:45,153 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36337:36337),(127.0.0.1/127.0.0.1:35575:35575)] 2024-11-10T04:22:45,156 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:22:45,156 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:45,156 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,156 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:22:45,160 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:22:45,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:22:45,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:22:45,164 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:22:45,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:22:45,166 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:22:45,166 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,167 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,168 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,169 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,169 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,170 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:22:45,171 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:22:45,174 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:22:45,174 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=855960, jitterRate=0.08840993046760559}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:22:45,175 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212565156Initializing all the Stores at 1731212565157 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212565157Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212565158 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212565158Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212565158Cleaning up temporary data from old regions at 1731212565169 (+11 ms)Region opened successfully at 1731212565175 (+6 ms) 2024-11-10T04:22:45,176 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:22:45,180 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445e5365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:22:45,181 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:22:45,181 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:22:45,181 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:22:45,181 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:22:45,181 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T04:22:45,182 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T04:22:45,182 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:22:45,184 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:22:45,185 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:22:45,187 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:22:45,187 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:22:45,188 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:22:45,191 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:22:45,192 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:22:45,192 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:22:45,194 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:22:45,195 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:22:45,197 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:22:45,199 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:22:45,200 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:22:45,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:45,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:22:45,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,203 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,40031,1731212564976, sessionid=0x101906acf480000, setting cluster-up flag (Was=false) 2024-11-10T04:22:45,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,212 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:22:45,213 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,223 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:22:45,224 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,226 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:22:45,228 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:45,228 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:22:45,228 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:22:45,229 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,40031,1731212564976 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:22:45,230 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212595232 2024-11-10T04:22:45,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:22:45,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:22:45,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:22:45,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:22:45,232 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:45,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:22:45,232 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:22:45,232 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:22:45,233 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,233 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:22:45,233 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:22:45,233 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:22:45,234 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,234 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:22:45,236 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:22:45,236 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:22:45,236 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212565236,5,FailOnTimeoutGroup] 2024-11-10T04:22:45,237 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212565236,5,FailOnTimeoutGroup] 2024-11-10T04:22:45,237 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,237 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:22:45,237 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,237 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:22:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:22:45,243 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:22:45,243 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6 2024-11-10T04:22:45,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:22:45,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:22:45,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:45,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:22:45,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:22:45,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:22:45,256 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:22:45,256 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,256 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(746): ClusterId : 7ba64496-0fa9-4701-bcde-2c2416623d24 2024-11-10T04:22:45,256 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:22:45,256 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,257 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:22:45,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:22:45,258 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:22:45,259 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:22:45,259 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:22:45,260 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:22:45,260 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:22:45,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740 2024-11-10T04:22:45,262 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:22:45,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740 2024-11-10T04:22:45,262 DEBUG [RS:0;d1d9e3766cb1:38007 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e55412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:22:45,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:22:45,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:22:45,264 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:22:45,266 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:22:45,268 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:22:45,268 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=784854, jitterRate=-0.0020071864128112793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:22:45,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212565251Initializing all the Stores at 1731212565252 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212565252Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212565252Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212565252Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212565252Cleaning up temporary data from old regions at 1731212565264 (+12 ms)Region opened successfully at 1731212565269 (+5 ms) 2024-11-10T04:22:45,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:22:45,269 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:22:45,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:22:45,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:22:45,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:22:45,270 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:22:45,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212565269Disabling compacts and flushes for region at 1731212565269Disabling writes for close at 1731212565269Writing region close event to WAL at 1731212565270 (+1 ms)Closed at 1731212565270 2024-11-10T04:22:45,271 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:45,271 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:22:45,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:22:45,273 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:22:45,274 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:22:45,281 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:38007 2024-11-10T04:22:45,282 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:22:45,282 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:22:45,282 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:22:45,283 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,40031,1731212564976 with port=38007, startcode=1731212565036 2024-11-10T04:22:45,283 DEBUG [RS:0;d1d9e3766cb1:38007 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:22:45,286 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38477, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:22:45,286 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40031 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,286 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40031 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,288 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6 2024-11-10T04:22:45,288 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40839 2024-11-10T04:22:45,288 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:22:45,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:22:45,291 DEBUG [RS:0;d1d9e3766cb1:38007 {}] zookeeper.ZKUtil(111): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,291 WARN [RS:0;d1d9e3766cb1:38007 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:22:45,291 INFO [RS:0;d1d9e3766cb1:38007 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:22:45,291 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,291 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,38007,1731212565036] 2024-11-10T04:22:45,296 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:22:45,300 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:22:45,300 INFO [RS:0;d1d9e3766cb1:38007 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:22:45,301 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,303 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:22:45,304 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:22:45,304 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,304 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,305 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:45,305 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:22:45,305 DEBUG [RS:0;d1d9e3766cb1:38007 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:22:45,306 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,306 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,306 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,306 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,306 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,306 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38007,1731212565036-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:22:45,330 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:22:45,331 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38007,1731212565036-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,331 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,331 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.Replication(171): d1d9e3766cb1,38007,1731212565036 started 2024-11-10T04:22:45,347 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,347 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,38007,1731212565036, RpcServer on d1d9e3766cb1/172.17.0.2:38007, sessionid=0x101906acf480001 2024-11-10T04:22:45,347 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:22:45,347 DEBUG [RS:0;d1d9e3766cb1:38007 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,347 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38007,1731212565036' 2024-11-10T04:22:45,347 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:22:45,348 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:22:45,349 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:22:45,349 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:22:45,349 DEBUG [RS:0;d1d9e3766cb1:38007 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,349 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38007,1731212565036' 2024-11-10T04:22:45,349 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:22:45,349 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:22:45,350 DEBUG [RS:0;d1d9e3766cb1:38007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:22:45,350 INFO [RS:0;d1d9e3766cb1:38007 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:22:45,350 INFO [RS:0;d1d9e3766cb1:38007 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:22:45,425 WARN [d1d9e3766cb1:40031 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:22:45,452 INFO [RS:0;d1d9e3766cb1:38007 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38007%2C1731212565036, suffix=, logDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036, archiveDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs, maxLogs=32 2024-11-10T04:22:45,453 INFO [RS:0;d1d9e3766cb1:38007 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 2024-11-10T04:22:45,461 INFO [RS:0;d1d9e3766cb1:38007 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 2024-11-10T04:22:45,466 DEBUG [RS:0;d1d9e3766cb1:38007 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36337:36337),(127.0.0.1/127.0.0.1:35575:35575)] 2024-11-10T04:22:45,675 DEBUG [d1d9e3766cb1:40031 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:22:45,676 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,678 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38007,1731212565036, state=OPENING 2024-11-10T04:22:45,679 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:22:45,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:22:45,681 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:22:45,681 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:45,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:45,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38007,1731212565036}] 2024-11-10T04:22:45,836 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:22:45,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41367, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:22:45,842 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:22:45,842 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:22:45,844 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38007%2C1731212565036.meta, suffix=.meta, logDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036, archiveDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs, maxLogs=32 2024-11-10T04:22:45,845 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta 2024-11-10T04:22:45,850 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta 2024-11-10T04:22:45,851 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35575:35575),(127.0.0.1/127.0.0.1:36337:36337)] 2024-11-10T04:22:45,854 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:22:45,855 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:22:45,855 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:22:45,855 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:22:45,855 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:22:45,855 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:45,855 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:22:45,855 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:22:45,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:22:45,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:22:45,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:22:45,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:22:45,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:22:45,861 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:22:45,861 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:22:45,863 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:22:45,863 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:45,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:22:45,864 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:22:45,864 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740 2024-11-10T04:22:45,866 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740 2024-11-10T04:22:45,867 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:22:45,867 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:22:45,868 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:22:45,869 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:22:45,870 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=718597, jitterRate=-0.08625707030296326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:22:45,871 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:22:45,872 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212565855Writing region info on filesystem at 1731212565855Initializing all the Stores at 1731212565856 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212565856Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212565857 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212565857Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212565857Cleaning up temporary data from old regions at 1731212565867 (+10 ms)Running coprocessor post-open hooks at 1731212565871 (+4 ms)Region opened successfully at 1731212565872 (+1 ms) 2024-11-10T04:22:45,873 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212565835 2024-11-10T04:22:45,876 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:22:45,877 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:22:45,878 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,879 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38007,1731212565036, state=OPEN 2024-11-10T04:22:45,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:22:45,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:22:45,884 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:45,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:45,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:22:45,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:22:45,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38007,1731212565036 in 202 msec 2024-11-10T04:22:45,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:22:45,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 615 msec 2024-11-10T04:22:45,891 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:22:45,892 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:22:45,893 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:22:45,893 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38007,1731212565036, seqNum=-1] 2024-11-10T04:22:45,893 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:22:45,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59095, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:22:45,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 673 msec 2024-11-10T04:22:45,901 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212565901, completionTime=-1 2024-11-10T04:22:45,901 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:22:45,901 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:22:45,903 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:22:45,903 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212625903 2024-11-10T04:22:45,903 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731212685903 2024-11-10T04:22:45,903 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-10T04:22:45,904 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40031,1731212564976-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,904 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40031,1731212564976-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,904 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40031,1731212564976-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,904 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:40031, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,904 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,904 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,906 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.832sec 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40031,1731212564976-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:22:45,908 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40031,1731212564976-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:22:45,911 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:22:45,911 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:22:45,911 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40031,1731212564976-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:45,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-10T04:22:45,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd7b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:22:45,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,40031,-1 for getting cluster id 2024-11-10T04:22:45,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:22:45,958 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ba64496-0fa9-4701-bcde-2c2416623d24' 2024-11-10T04:22:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:22:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ba64496-0fa9-4701-bcde-2c2416623d24" 2024-11-10T04:22:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f54b35c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:22:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,40031,-1] 2024-11-10T04:22:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:22:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:22:45,962 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:22:45,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5757a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:22:45,963 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:22:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38007,1731212565036, seqNum=-1] 2024-11-10T04:22:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:22:45,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:22:45,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:45,968 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:45,972 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T04:22:45,994 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:22:45,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:45,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:45,995 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:22:45,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:22:45,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:22:45,995 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:22:45,995 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:22:45,996 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33499 2024-11-10T04:22:45,997 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33499 connecting to ZooKeeper ensemble=127.0.0.1:60824 2024-11-10T04:22:45,998 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:46,000 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:22:46,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334990x0, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:22:46,005 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33499-0x101906acf480002 connected 2024-11-10T04:22:46,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-10T04:22:46,005 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-10T04:22:46,006 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:22:46,006 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:22:46,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:22:46,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:22:46,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33499 2024-11-10T04:22:46,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33499 2024-11-10T04:22:46,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33499 2024-11-10T04:22:46,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33499 2024-11-10T04:22:46,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33499 2024-11-10T04:22:46,013 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(746): ClusterId : 7ba64496-0fa9-4701-bcde-2c2416623d24 2024-11-10T04:22:46,013 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:22:46,016 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:22:46,016 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:22:46,018 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:22:46,019 DEBUG [RS:1;d1d9e3766cb1:33499 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d6522a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:22:46,056 DEBUG [RS:1;d1d9e3766cb1:33499 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d1d9e3766cb1:33499 2024-11-10T04:22:46,056 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:22:46,056 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:22:46,056 DEBUG [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:22:46,057 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,40031,1731212564976 with port=33499, startcode=1731212565994 2024-11-10T04:22:46,057 DEBUG [RS:1;d1d9e3766cb1:33499 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:22:46,059 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46731, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:22:46,060 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40031 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,33499,1731212565994 2024-11-10T04:22:46,060 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40031 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,33499,1731212565994 2024-11-10T04:22:46,062 DEBUG [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6 2024-11-10T04:22:46,062 DEBUG [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40839 2024-11-10T04:22:46,062 DEBUG [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:22:46,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:22:46,064 DEBUG [RS:1;d1d9e3766cb1:33499 {}] zookeeper.ZKUtil(111): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,33499,1731212565994 2024-11-10T04:22:46,064 WARN [RS:1;d1d9e3766cb1:33499 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:22:46,064 INFO [RS:1;d1d9e3766cb1:33499 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:22:46,064 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,33499,1731212565994] 2024-11-10T04:22:46,064 DEBUG [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994 2024-11-10T04:22:46,068 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:22:46,070 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:22:46,071 INFO [RS:1;d1d9e3766cb1:33499 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:22:46,071 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,071 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:22:46,072 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:22:46,072 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,072 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,072 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,072 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,072 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,072 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,072 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:22:46,072 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,073 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,073 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,073 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,073 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,073 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:22:46,073 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:22:46,073 DEBUG [RS:1;d1d9e3766cb1:33499 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:22:46,075 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,075 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,075 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,075 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,075 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,075 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,33499,1731212565994-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:22:46,080 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:46,085 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:46,096 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:22:46,096 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,33499,1731212565994-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,096 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,096 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.Replication(171): d1d9e3766cb1,33499,1731212565994 started 2024-11-10T04:22:46,112 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:22:46,112 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,33499,1731212565994, RpcServer on d1d9e3766cb1/172.17.0.2:33499, sessionid=0x101906acf480002 2024-11-10T04:22:46,112 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:22:46,112 DEBUG [RS:1;d1d9e3766cb1:33499 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,33499,1731212565994 2024-11-10T04:22:46,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;d1d9e3766cb1:33499,5,FailOnTimeoutGroup] 2024-11-10T04:22:46,112 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,33499,1731212565994' 2024-11-10T04:22:46,112 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:22:46,112 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-11-10T04:22:46,112 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:22:46,113 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:22:46,113 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:22:46,113 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:22:46,113 DEBUG [RS:1;d1d9e3766cb1:33499 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,33499,1731212565994 2024-11-10T04:22:46,113 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,33499,1731212565994' 2024-11-10T04:22:46,113 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:22:46,114 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:22:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,40031,1731212564976 2024-11-10T04:22:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ae12d46 2024-11-10T04:22:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:22:46,114 DEBUG [RS:1;d1d9e3766cb1:33499 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:22:46,114 INFO [RS:1;d1d9e3766cb1:33499 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:22:46,114 INFO [RS:1;d1d9e3766cb1:33499 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:22:46,116 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:22:46,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40031 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T04:22:46,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T04:22:46,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:22:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40031 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T04:22:46,120 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:22:46,120 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:46,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-10T04:22:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:22:46,121 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:22:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741835_1011 (size=393) 2024-11-10T04:22:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741835_1011 (size=393) 2024-11-10T04:22:46,130 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df8778146310ba3408609557bb98b9e5, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6 2024-11-10T04:22:46,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41831 is added to blk_1073741836_1012 (size=76) 2024-11-10T04:22:46,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44537 is added to blk_1073741836_1012 (size=76) 2024-11-10T04:22:46,138 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:46,138 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing df8778146310ba3408609557bb98b9e5, disabling compactions & flushes 2024-11-10T04:22:46,138 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:46,138 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:46,138 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. after waiting 0 ms 2024-11-10T04:22:46,138 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:46,138 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:46,138 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for df8778146310ba3408609557bb98b9e5: Waiting for close lock at 1731212566138Disabling compacts and flushes for region at 1731212566138Disabling writes for close at 1731212566138Writing region close event to WAL at 1731212566138Closed at 1731212566138 2024-11-10T04:22:46,140 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:22:46,140 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731212566140"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212566140"}]},"ts":"1731212566140"} 2024-11-10T04:22:46,143 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:22:46,144 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:22:46,144 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212566144"}]},"ts":"1731212566144"} 2024-11-10T04:22:46,147 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-10T04:22:46,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=df8778146310ba3408609557bb98b9e5, ASSIGN}] 2024-11-10T04:22:46,149 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=df8778146310ba3408609557bb98b9e5, ASSIGN 2024-11-10T04:22:46,150 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=df8778146310ba3408609557bb98b9e5, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,38007,1731212565036; forceNewPlan=false, retain=false 2024-11-10T04:22:46,216 INFO [RS:1;d1d9e3766cb1:33499 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C33499%2C1731212565994, suffix=, logDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994, archiveDir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs, maxLogs=32 2024-11-10T04:22:46,217 INFO [RS:1;d1d9e3766cb1:33499 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 2024-11-10T04:22:46,223 INFO [RS:1;d1d9e3766cb1:33499 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 2024-11-10T04:22:46,224 DEBUG [RS:1;d1d9e3766cb1:33499 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35575:35575),(127.0.0.1/127.0.0.1:36337:36337)] 2024-11-10T04:22:46,301 INFO [d1d9e3766cb1:40031 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T04:22:46,302 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df8778146310ba3408609557bb98b9e5, regionState=OPENING, regionLocation=d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:46,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=df8778146310ba3408609557bb98b9e5, ASSIGN because future has completed 2024-11-10T04:22:46,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df8778146310ba3408609557bb98b9e5, server=d1d9e3766cb1,38007,1731212565036}] 2024-11-10T04:22:46,462 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:46,463 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => df8778146310ba3408609557bb98b9e5, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:22:46,463 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,463 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:22:46,464 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,464 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,465 INFO [StoreOpener-df8778146310ba3408609557bb98b9e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,467 INFO [StoreOpener-df8778146310ba3408609557bb98b9e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df8778146310ba3408609557bb98b9e5 columnFamilyName info 2024-11-10T04:22:46,467 DEBUG [StoreOpener-df8778146310ba3408609557bb98b9e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:22:46,467 INFO [StoreOpener-df8778146310ba3408609557bb98b9e5-1 {}] regionserver.HStore(327): Store=df8778146310ba3408609557bb98b9e5/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:22:46,467 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,468 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,468 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,469 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,469 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,470 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,473 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:22:46,473 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened df8778146310ba3408609557bb98b9e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=821419, jitterRate=0.04448850452899933}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:22:46,473 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df8778146310ba3408609557bb98b9e5 2024-11-10T04:22:46,474 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for df8778146310ba3408609557bb98b9e5: Running coprocessor pre-open hook at 1731212566464Writing region info on filesystem at 1731212566464Initializing all the Stores at 1731212566465 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212566465Cleaning up temporary data from old regions at 1731212566469 (+4 ms)Running coprocessor post-open hooks at 1731212566473 (+4 ms)Region opened successfully at 1731212566474 (+1 ms) 2024-11-10T04:22:46,475 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5., pid=6, masterSystemTime=1731212566458 2024-11-10T04:22:46,478 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:46,478 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:46,479 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df8778146310ba3408609557bb98b9e5, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38007,1731212565036 2024-11-10T04:22:46,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df8778146310ba3408609557bb98b9e5, server=d1d9e3766cb1,38007,1731212565036 because future has completed 2024-11-10T04:22:46,482 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40031 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d1d9e3766cb1,38007,1731212565036, table=TestLogRolling-testLogRollOnDatanodeDeath, region=df8778146310ba3408609557bb98b9e5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T04:22:46,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T04:22:46,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure df8778146310ba3408609557bb98b9e5, server=d1d9e3766cb1,38007,1731212565036 in 178 msec 2024-11-10T04:22:46,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T04:22:46,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=df8778146310ba3408609557bb98b9e5, ASSIGN in 339 msec 2024-11-10T04:22:46,490 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:22:46,490 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212566490"}]},"ts":"1731212566490"} 2024-11-10T04:22:46,492 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-10T04:22:46,493 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:22:46,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 376 msec 2024-11-10T04:22:46,602 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:22:46,603 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:46,623 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:46,625 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:46,625 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:51,296 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-10T04:22:51,965 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:22:51,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:51,984 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:51,987 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:51,988 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:22:55,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T04:22:55,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-10T04:22:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:22:56,218 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-10T04:22:56,218 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-10T04:22:56,222 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T04:22:56,222 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:22:56,245 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:56,248 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:56,249 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:56,249 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:56,249 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:22:56,250 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51f59516{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:56,251 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20e1b523{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:56,369 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7041a5bb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir/jetty-localhost-34371-hadoop-hdfs-3_4_1-tests_jar-_-any-9797515598942938390/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:56,370 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2cccc2fb{HTTP/1.1, (http/1.1)}{localhost:34371} 2024-11-10T04:22:56,370 INFO [Time-limited test {}] server.Server(415): Started @115915ms 2024-11-10T04:22:56,371 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:22:56,419 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:56,424 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:56,432 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:56,432 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:56,432 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:22:56,433 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3afd309b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:56,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32dac098{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:56,480 WARN [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data5/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:56,480 WARN [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data6/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:56,504 WARN [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:22:56,507 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x78a651f2f096919e with lease ID 0x88ed51acd8d0e3a2: Processing first storage report for DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0 from datanode DatanodeRegistration(127.0.0.1:37841, datanodeUuid=c21f9c78-57a6-4de7-b324-9392df845799, infoPort=41783, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:56,507 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x78a651f2f096919e with lease ID 0x88ed51acd8d0e3a2: from storage DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0 node DatanodeRegistration(127.0.0.1:37841, datanodeUuid=c21f9c78-57a6-4de7-b324-9392df845799, infoPort=41783, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:56,508 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x78a651f2f096919e with lease ID 0x88ed51acd8d0e3a2: Processing first storage report for DS-ca12d85c-3abe-4b50-92b7-d68e07f6144d from datanode DatanodeRegistration(127.0.0.1:37841, datanodeUuid=c21f9c78-57a6-4de7-b324-9392df845799, infoPort=41783, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:56,508 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x78a651f2f096919e with lease ID 0x88ed51acd8d0e3a2: from storage DS-ca12d85c-3abe-4b50-92b7-d68e07f6144d node DatanodeRegistration(127.0.0.1:37841, datanodeUuid=c21f9c78-57a6-4de7-b324-9392df845799, infoPort=41783, infoSecurePort=0, ipcPort=37345, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:56,565 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@567c82bb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir/jetty-localhost-39193-hadoop-hdfs-3_4_1-tests_jar-_-any-15885356639662657375/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:56,566 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@203cb204{HTTP/1.1, (http/1.1)}{localhost:39193} 2024-11-10T04:22:56,566 INFO [Time-limited test {}] server.Server(415): Started @116111ms 2024-11-10T04:22:56,568 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:22:56,623 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:22:56,628 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:22:56,634 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:22:56,634 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:22:56,634 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:22:56,634 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78e445ac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:22:56,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f55aa3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:22:56,684 WARN [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data7/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:56,684 WARN [Thread-865 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data8/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:56,709 WARN [Thread-844 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:22:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7d1ec132d29aa4e with lease ID 0x88ed51acd8d0e3a3: Processing first storage report for DS-080e8f71-33a6-41d0-a7cb-6135b647691d from datanode DatanodeRegistration(127.0.0.1:40843, datanodeUuid=56968266-a2d8-4fe0-9bdb-f2c35cf01087, infoPort=44237, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7d1ec132d29aa4e with lease ID 0x88ed51acd8d0e3a3: from storage DS-080e8f71-33a6-41d0-a7cb-6135b647691d node DatanodeRegistration(127.0.0.1:40843, datanodeUuid=56968266-a2d8-4fe0-9bdb-f2c35cf01087, infoPort=44237, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T04:22:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7d1ec132d29aa4e with lease ID 0x88ed51acd8d0e3a3: Processing first storage report for DS-1cf99454-256e-4cce-8c3b-1a33a3707d80 from datanode DatanodeRegistration(127.0.0.1:40843, datanodeUuid=56968266-a2d8-4fe0-9bdb-f2c35cf01087, infoPort=44237, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7d1ec132d29aa4e with lease ID 0x88ed51acd8d0e3a3: from storage DS-1cf99454-256e-4cce-8c3b-1a33a3707d80 node DatanodeRegistration(127.0.0.1:40843, datanodeUuid=56968266-a2d8-4fe0-9bdb-f2c35cf01087, infoPort=44237, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:56,770 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c82feab{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir/jetty-localhost-33687-hadoop-hdfs-3_4_1-tests_jar-_-any-6521227177718254821/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:56,770 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74865863{HTTP/1.1, (http/1.1)}{localhost:33687} 2024-11-10T04:22:56,770 INFO [Time-limited test {}] server.Server(415): Started @116315ms 2024-11-10T04:22:56,771 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:22:56,869 WARN [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:56,869 WARN [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9/current/BP-1586179451-172.17.0.2-1731212564275/current, will proceed with Du for space computation calculation, 2024-11-10T04:22:56,897 WARN [Thread-879 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:22:56,900 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6e1a94dfae4bdd4d with lease ID 0x88ed51acd8d0e3a4: Processing first storage report for DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff from datanode DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:56,900 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6e1a94dfae4bdd4d with lease ID 0x88ed51acd8d0e3a4: from storage DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff node DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6e1a94dfae4bdd4d with lease ID 0x88ed51acd8d0e3a4: Processing first storage report for DS-827c97f7-e20d-4eed-9679-c249b83975a2 from datanode DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275) 2024-11-10T04:22:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6e1a94dfae4bdd4d with lease ID 0x88ed51acd8d0e3a4: from storage DS-827c97f7-e20d-4eed-9679-c249b83975a2 node DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:22:56,995 WARN [ResponseProcessor for block BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:56,995 WARN [ResponseProcessor for block BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:56,995 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:22:56,995 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:22:56,996 WARN [ResponseProcessor for block BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:56,998 WARN [ResponseProcessor for block BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013 java.io.IOException: Bad response ERROR for BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:56,998 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:22:56,999 WARN [PacketResponder: BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41831] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,000 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta block BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:22:56,999 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:48766 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41831:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48766 dst: /127.0.0.1:41831 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,000 WARN [PacketResponder: BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41831] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,000 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43666 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44537:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43666 dst: /127.0.0.1:44537 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,000 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43658 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44537:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43658 dst: /127.0.0.1:44537 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,000 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1617658036_22 at /127.0.0.1:48716 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41831:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48716 dst: /127.0.0.1:41831 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,000 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1925875247_22 at /127.0.0.1:48800 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:41831:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48800 dst: /127.0.0.1:41831 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,001 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21d5e4af{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:57,001 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:48770 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41831:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48770 dst: /127.0.0.1:41831 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,001 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1925875247_22 at /127.0.0.1:43686 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44537:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43686 dst: /127.0.0.1:44537 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,001 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1617658036_22 at /127.0.0.1:43626 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44537:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43626 dst: /127.0.0.1:44537 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,001 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@145c0180{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:57,002 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:57,002 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7720beab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:57,002 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69bbaec1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:57,005 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:22:57,005 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:22:57,005 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:22:57,005 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1586179451-172.17.0.2-1731212564275 (Datanode Uuid 36bda998-80e7-46b9-9d7b-4fc68015f300) service to localhost/127.0.0.1:40839 2024-11-10T04:22:57,006 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:57,006 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:57,007 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:22:57,008 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta block BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,007 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1925875247_22 at /127.0.0.1:41770 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44537:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41770 dst: /127.0.0.1:44537 java.io.IOException: The stream is closed at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,008 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,008 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1617658036_22 at /127.0.0.1:41780 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44537:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41780 dst: /127.0.0.1:44537 java.io.IOException: The stream is closed at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,008 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,008 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:41792 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44537:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41792 dst: /127.0.0.1:44537 java.io.IOException: The stream is closed at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:57,009 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@ab5393f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:57,013 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47f82e76{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:57,013 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:57,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30e7c448{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:57,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@257cf4bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:57,015 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:22:57,015 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1586179451-172.17.0.2-1731212564275 (Datanode Uuid cc08ad11-27db-4910-a156-7a697eb22d59) service to localhost/127.0.0.1:40839 2024-11-10T04:22:57,015 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:22:57,015 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:22:57,016 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data1/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:57,016 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data2/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:57,016 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:22:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5., hostname=d1d9e3766cb1,38007,1731212565036, seqNum=2] 2024-11-10T04:22:57,023 ERROR [FSHLog-0-hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6-prefix:d1d9e3766cb1,38007,1731212565036 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,023 WARN [FSHLog-0-hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6-prefix:d1d9e3766cb1,38007,1731212565036 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,024 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,024 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38007%2C1731212565036:(num 1731212565453) roll requested 2024-11-10T04:22:57,024 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 2024-11-10T04:22:57,028 WARN [Thread-904 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,028 WARN [Thread-904 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:22:57,028 WARN [Thread-904 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741838_1018 2024-11-10T04:22:57,031 WARN [Thread-904 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:22:57,034 WARN [Thread-904 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1019 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,035 WARN [Thread-904 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:22:57,035 WARN [Thread-904 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741839_1019 2024-11-10T04:22:57,035 WARN [Thread-904 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:22:57,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:57,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:57,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:57,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:57,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:22:57,041 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 2024-11-10T04:22:57,042 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,042 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:57,043 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-10T04:22:57,044 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-10T04:22:57,044 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 2024-11-10T04:22:57,047 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41783:41783),(127.0.0.1/127.0.0.1:44237:44237)] 2024-11-10T04:22:57,047 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 is not closed yet, will try archiving it next time 2024-11-10T04:22:57,047 WARN [IPC Server handler 4 on default port 40839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 has not been closed. Lease recovery is in progress. RecoveryId = 1021 for block blk_1073741833_1009 2024-11-10T04:22:57,051 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 after 5ms 2024-11-10T04:22:57,124 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:58,074 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:59,047 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:59,049 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 2024-11-10T04:22:59,049 WARN [ResponseProcessor for block BP-1586179451-172.17.0.2-1731212564275:blk_1073741840_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1586179451-172.17.0.2-1731212564275:blk_1073741840_1020 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:22:59,050 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741840_1020 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:22:59,051 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:60410 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:37841:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60410 dst: /127.0.0.1:37841 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:59,051 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:49040 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:40843:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49040 dst: /127.0.0.1:40843 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:22:59,054 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7041a5bb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:22:59,055 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2cccc2fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:22:59,055 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:22:59,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20e1b523{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:22:59,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51f59516{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,STOPPED} 2024-11-10T04:22:59,056 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:22:59,056 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:22:59,056 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:22:59,056 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1586179451-172.17.0.2-1731212564275 (Datanode Uuid c21f9c78-57a6-4de7-b324-9392df845799) service to localhost/127.0.0.1:40839 2024-11-10T04:22:59,057 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data5/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:59,057 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data6/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:22:59,057 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:22:59,125 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:00,074 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:01,048 WARN [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]] 2024-11-10T04:23:01,048 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:01,048 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38007%2C1731212565036:(num 1731212577024) roll requested 2024-11-10T04:23:01,048 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.1731212581048 2024-11-10T04:23:01,052 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 after 4008ms 2024-11-10T04:23:01,053 WARN [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41831 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:01,053 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55302 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741841_1023 to mirror 127.0.0.1:41831 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:01,053 WARN [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:01,053 WARN [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741841_1023 2024-11-10T04:23:01,053 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55302 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:01,053 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55302 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55302 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:01,054 WARN [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:01,056 WARN [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37841 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:01,056 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55318 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741842_1024] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741842_1024 to mirror 127.0.0.1:37841 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:01,056 WARN [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:01,056 WARN [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741842_1024 2024-11-10T04:23:01,056 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55318 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741842_1024] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:01,057 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55318 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55318 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:01,057 WARN [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:01,059 WARN [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:01,059 WARN [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:01,059 WARN [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741843_1025 2024-11-10T04:23:01,060 WARN [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:01,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:01,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:01,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:01,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:01,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:01,065 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212581048 2024-11-10T04:23:01,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40843 is added to blk_1073741840_1022 (size=2431) 2024-11-10T04:23:01,069 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T04:23:01,074 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44237:44237),(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T04:23:01,074 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 is not closed yet, will try archiving it next time 2024-11-10T04:23:01,074 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 is not closed yet, will try archiving it next time 2024-11-10T04:23:01,125 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:01,468 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 is not closed yet, will try archiving it next time 2024-11-10T04:23:02,075 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:02,724 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@11b34848[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40843, datanodeUuid=56968266-a2d8-4fe0-9bdb-f2c35cf01087, infoPort=44237, infoSecurePort=0, ipcPort=39739, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741840_1022 to 127.0.0.1:44537 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:03,074 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,076 WARN [ResponseProcessor for block BP-1586179451-172.17.0.2-1731212564275:blk_1073741844_1026 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1586179451-172.17.0.2-1731212564275:blk_1073741844_1026 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,076 WARN [DataStreamer for file /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212581048 block BP-1586179451-172.17.0.2-1731212564275:blk_1073741844_1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:03,077 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55326 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55326 dst: /127.0.0.1:43793 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:03,077 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55666 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:40843:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55666 dst: /127.0.0.1:40843 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:03,079 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@567c82bb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:03,080 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@203cb204{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:03,080 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:03,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32dac098{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:03,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3afd309b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:03,082 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:23:03,082 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:23:03,082 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1586179451-172.17.0.2-1731212564275 (Datanode Uuid 56968266-a2d8-4fe0-9bdb-f2c35cf01087) service to localhost/127.0.0.1:40839 2024-11-10T04:23:03,082 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:23:03,082 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data7/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:03,083 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data8/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:03,083 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:23:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38007 {}] regionserver.HRegion(8855): Flush requested on df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:03,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df8778146310ba3408609557bb98b9e5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:23:03,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/15b104b0aa2346aca82cdfd3557dc209 is 1080, key is row0002/info:/1731212579059/Put/seqid=0 2024-11-10T04:23:03,116 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,116 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:03,116 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741845_1028 2024-11-10T04:23:03,118 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:03,120 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,120 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:03,120 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741846_1029 2024-11-10T04:23:03,121 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:03,123 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41831 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,123 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55340 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741847_1030 to mirror 127.0.0.1:41831 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:03,124 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:03,124 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55340 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:03,124 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741847_1030 2024-11-10T04:23:03,124 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55340 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55340 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:03,124 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:03,126 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,126 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,126 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:03,126 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741848_1031 2024-11-10T04:23:03,127 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:03,128 WARN [IPC Server handler 0 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:03,128 WARN [IPC Server handler 0 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:03,128 WARN [IPC Server handler 0 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:03,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741849_1032 (size=10347) 2024-11-10T04:23:03,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/15b104b0aa2346aca82cdfd3557dc209 2024-11-10T04:23:03,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/15b104b0aa2346aca82cdfd3557dc209 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/15b104b0aa2346aca82cdfd3557dc209 2024-11-10T04:23:03,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/15b104b0aa2346aca82cdfd3557dc209, entries=5, sequenceid=11, filesize=10.1 K 2024-11-10T04:23:03,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for df8778146310ba3408609557bb98b9e5 in 463ms, sequenceid=11, compaction requested=false 2024-11-10T04:23:03,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38007 {}] regionserver.HRegion(8855): Flush requested on df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:03,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df8778146310ba3408609557bb98b9e5 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-11-10T04:23:03,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/abaf90768f9f4daaa780ce350ce0822c is 1080, key is row0007/info:/1731212583094/Put/seqid=0 2024-11-10T04:23:03,735 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,735 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:03,735 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741850_1033 2024-11-10T04:23:03,736 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:03,737 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,738 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:03,738 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741851_1034 2024-11-10T04:23:03,738 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:03,740 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,740 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:03,740 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741852_1035 2024-11-10T04:23:03,740 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:03,743 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40843 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:03,743 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55372 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741853_1036 to mirror 127.0.0.1:40843 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:03,743 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:03,743 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741853_1036 2024-11-10T04:23:03,743 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55372 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:03,744 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55372 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55372 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:03,744 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:03,745 WARN [IPC Server handler 0 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:03,745 WARN [IPC Server handler 0 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:03,745 WARN [IPC Server handler 0 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:03,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741854_1037 (size=12506) 2024-11-10T04:23:04,075 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:04,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/abaf90768f9f4daaa780ce350ce0822c 2024-11-10T04:23:04,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/abaf90768f9f4daaa780ce350ce0822c as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c 2024-11-10T04:23:04,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c, entries=7, sequenceid=24, filesize=12.2 K 2024-11-10T04:23:04,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for df8778146310ba3408609557bb98b9e5 in 445ms, sequenceid=24, compaction requested=false 2024-11-10T04:23:04,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:04,174 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-11-10T04:23:04,174 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:04,174 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c because midkey is the same as first or last row 2024-11-10T04:23:05,075 WARN [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]] 2024-11-10T04:23:05,075 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,075 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38007%2C1731212565036:(num 1731212581048) roll requested 2024-11-10T04:23:05,076 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.1731212585075 2024-11-10T04:23:05,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55390 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741855_1038 to mirror 127.0.0.1:44537 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,083 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44537 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,083 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55390 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:05,083 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:05,084 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741855_1038 2024-11-10T04:23:05,084 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55390 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55390 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,084 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:05,086 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,086 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:05,086 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741856_1039 2024-11-10T04:23:05,087 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:05,095 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40843 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,095 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55396 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741857_1040 to mirror 127.0.0.1:40843 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,096 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:05,096 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741857_1040 2024-11-10T04:23:05,096 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55396 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:05,096 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55396 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55396 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,096 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:05,098 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,098 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:05,098 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741858_1041 2024-11-10T04:23:05,098 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:05,099 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:05,099 WARN [IPC Server handler 4 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:05,099 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:05,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:05,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:05,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:05,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:05,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:05,103 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212581048 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212585075 2024-11-10T04:23:05,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741844_1027 (size=25992) 2024-11-10T04:23:05,105 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T04:23:05,105 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 is not closed yet, will try archiving it next time 2024-11-10T04:23:05,105 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212581048 is not closed yet, will try archiving it next time 2024-11-10T04:23:05,106 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs/d1d9e3766cb1%2C38007%2C1731212565036.1731212577024 2024-11-10T04:23:05,126 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38007 {}] regionserver.HRegion(8855): Flush requested on df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:05,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df8778146310ba3408609557bb98b9e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T04:23:05,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/03adef152f8e4efe9214892bd8d376a3 is 1079, key is tmprow/info:/1731212585148/Put/seqid=0 2024-11-10T04:23:05,157 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,157 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:05,157 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741860_1043 2024-11-10T04:23:05,158 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:05,159 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,159 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:05,159 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741861_1044 2024-11-10T04:23:05,160 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:05,161 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,161 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:05,161 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741862_1045 2024-11-10T04:23:05,161 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:05,164 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44537 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,164 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55412 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741863_1046 to mirror 127.0.0.1:44537 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,164 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:05,164 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741863_1046 2024-11-10T04:23:05,164 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55412 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:05,164 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55412 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55412 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,165 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:05,165 WARN [IPC Server handler 3 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:05,165 WARN [IPC Server handler 3 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:05,165 WARN [IPC Server handler 3 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:05,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741864_1047 (size=6027) 2024-11-10T04:23:05,505 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 is not closed yet, will try archiving it next time 2024-11-10T04:23:05,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/03adef152f8e4efe9214892bd8d376a3 2024-11-10T04:23:05,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/03adef152f8e4efe9214892bd8d376a3 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/03adef152f8e4efe9214892bd8d376a3 2024-11-10T04:23:05,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/03adef152f8e4efe9214892bd8d376a3, entries=1, sequenceid=34, filesize=5.9 K 2024-11-10T04:23:05,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for df8778146310ba3408609557bb98b9e5 in 434ms, sequenceid=34, compaction requested=true 2024-11-10T04:23:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-11-10T04:23:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c because midkey is the same as first or last row 2024-11-10T04:23:05,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df8778146310ba3408609557bb98b9e5:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:23:05,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:23:05,585 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:23:05,586 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:23:05,586 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1541): df8778146310ba3408609557bb98b9e5/info is initiating minor compaction (all files) 2024-11-10T04:23:05,586 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df8778146310ba3408609557bb98b9e5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:05,586 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/15b104b0aa2346aca82cdfd3557dc209, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/03adef152f8e4efe9214892bd8d376a3] into tmpdir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp, totalSize=28.2 K 2024-11-10T04:23:05,587 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15b104b0aa2346aca82cdfd3557dc209, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731212579059 2024-11-10T04:23:05,587 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting abaf90768f9f4daaa780ce350ce0822c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731212583094 2024-11-10T04:23:05,588 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03adef152f8e4efe9214892bd8d376a3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731212585148 2024-11-10T04:23:05,601 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df8778146310ba3408609557bb98b9e5#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:23:05,602 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/5a4824dc206e471ba64a11383a1d4c39 is 1080, key is row0002/info:/1731212579059/Put/seqid=0 2024-11-10T04:23:05,603 WARN [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,603 WARN [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:05,603 WARN [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741865_1048 2024-11-10T04:23:05,604 WARN [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:05,605 WARN [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,605 WARN [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:05,605 WARN [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741866_1049 2024-11-10T04:23:05,606 WARN [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:05,606 WARN [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,607 WARN [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:05,607 WARN [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741867_1050 2024-11-10T04:23:05,607 WARN [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:05,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55450 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741868_1051 to mirror 127.0.0.1:37841 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,609 WARN [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37841 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:05,610 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55450 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:05,610 WARN [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:05,610 WARN [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741868_1051 2024-11-10T04:23:05,610 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55450 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55450 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,610 WARN [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:05,611 WARN [IPC Server handler 1 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:05,611 WARN [IPC Server handler 1 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:05,611 WARN [IPC Server handler 1 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:05,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741869_1052 (size=17994) 2024-11-10T04:23:05,621 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/5a4824dc206e471ba64a11383a1d4c39 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 2024-11-10T04:23:05,628 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df8778146310ba3408609557bb98b9e5/info of df8778146310ba3408609557bb98b9e5 into 5a4824dc206e471ba64a11383a1d4c39(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:23:05,628 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:05,629 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5., storeName=df8778146310ba3408609557bb98b9e5/info, priority=13, startTime=1731212585584; duration=0sec 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 because midkey is the same as first or last row 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 because midkey is the same as first or last row 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 because midkey is the same as first or last row 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:23:05,629 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df8778146310ba3408609557bb98b9e5:info 2024-11-10T04:23:05,900 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@30fd4369[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741849_1032 to 127.0.0.1:37841 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:05,900 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@54803134[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741854_1037 to 127.0.0.1:40843 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:06,076 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38007 {}] regionserver.HRegion(8855): Flush requested on df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:06,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df8778146310ba3408609557bb98b9e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T04:23:06,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/194e74659be94a69b5209d4c9fdba1b7 is 1079, key is tmprow/info:/1731212586569/Put/seqid=0 2024-11-10T04:23:06,577 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:06,577 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:06,577 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741870_1053 2024-11-10T04:23:06,577 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:06,579 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:06,579 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:06,579 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741871_1054 2024-11-10T04:23:06,579 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:06,580 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:06,581 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:06,581 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741872_1055 2024-11-10T04:23:06,581 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:06,582 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:06,582 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:06,582 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741873_1056 2024-11-10T04:23:06,583 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:06,584 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:06,584 WARN [IPC Server handler 4 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:06,584 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:06,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741874_1057 (size=6027) 2024-11-10T04:23:06,900 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@30fd4369[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741864_1047 to 127.0.0.1:40843 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:06,900 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@54803134[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741844_1027 to 127.0.0.1:41831 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:06,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/194e74659be94a69b5209d4c9fdba1b7 2024-11-10T04:23:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/194e74659be94a69b5209d4c9fdba1b7 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/194e74659be94a69b5209d4c9fdba1b7 2024-11-10T04:23:07,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/194e74659be94a69b5209d4c9fdba1b7, entries=1, sequenceid=45, filesize=5.9 K 2024-11-10T04:23:07,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for df8778146310ba3408609557bb98b9e5 in 432ms, sequenceid=45, compaction requested=false 2024-11-10T04:23:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-11-10T04:23:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 because midkey is the same as first or last row 2024-11-10T04:23:07,106 WARN [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]] 2024-11-10T04:23:07,106 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:07,106 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38007%2C1731212565036:(num 1731212585075) roll requested 2024-11-10T04:23:07,106 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.1731212587106 2024-11-10T04:23:07,109 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:07,109 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:07,110 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741875_1058 2024-11-10T04:23:07,110 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:07,111 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:07,112 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:07,112 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741876_1059 2024-11-10T04:23:07,112 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:07,113 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:07,114 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:07,114 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741877_1060 2024-11-10T04:23:07,114 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:07,116 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55478 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741878_1061] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741878_1061 to mirror 127.0.0.1:37841 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:07,116 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37841 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:07,117 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55478 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741878_1061] {}] datanode.BlockReceiver(316): Block 1073741878 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:07,117 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:07,117 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741878_1061 2024-11-10T04:23:07,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55478 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741878_1061] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55478 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:07,117 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:07,118 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:07,118 WARN [IPC Server handler 4 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:07,118 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:07,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:07,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:07,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:07,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:07,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:07,121 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212585075 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212587106 2024-11-10T04:23:07,122 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T04:23:07,122 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 is not closed yet, will try archiving it next time 2024-11-10T04:23:07,122 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212585075 is not closed yet, will try archiving it next time 2024-11-10T04:23:07,122 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212581048 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs/d1d9e3766cb1%2C38007%2C1731212565036.1731212581048 2024-11-10T04:23:07,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741859_1042 (size=13591) 2024-11-10T04:23:07,123 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 is not closed yet, will try archiving it next time 2024-11-10T04:23:07,126 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38007 {}] regionserver.HRegion(8855): Flush requested on df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:07,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df8778146310ba3408609557bb98b9e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T04:23:07,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/0aea2e62f1464e168ad1afbe0840d72c is 1079, key is tmprow/info:/1731212587990/Put/seqid=0 2024-11-10T04:23:07,999 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41831 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:07,999 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55484 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741880_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741880_1063 to mirror 127.0.0.1:41831 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:07,999 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:07,999 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55484 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741880_1063] {}] datanode.BlockReceiver(316): Block 1073741880 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:07,999 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741880_1063 2024-11-10T04:23:07,999 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55484 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741880_1063] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55484 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:08,000 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:08,001 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,001 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:08,001 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741881_1064 2024-11-10T04:23:08,002 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:08,003 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,003 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:08,003 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741882_1065 2024-11-10T04:23:08,004 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:08,005 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,005 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:08,005 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741883_1066 2024-11-10T04:23:08,006 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:08,006 WARN [IPC Server handler 0 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:08,006 WARN [IPC Server handler 0 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:08,007 WARN [IPC Server handler 0 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:08,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741884_1067 (size=6027) 2024-11-10T04:23:08,076 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/0aea2e62f1464e168ad1afbe0840d72c 2024-11-10T04:23:08,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/0aea2e62f1464e168ad1afbe0840d72c as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/0aea2e62f1464e168ad1afbe0840d72c 2024-11-10T04:23:08,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/0aea2e62f1464e168ad1afbe0840d72c, entries=1, sequenceid=55, filesize=5.9 K 2024-11-10T04:23:08,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for df8778146310ba3408609557bb98b9e5 in 438ms, sequenceid=55, compaction requested=true 2024-11-10T04:23:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K 2024-11-10T04:23:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 because midkey is the same as first or last row 2024-11-10T04:23:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df8778146310ba3408609557bb98b9e5:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:23:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:23:08,430 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:23:08,432 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:23:08,432 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1541): df8778146310ba3408609557bb98b9e5/info is initiating minor compaction (all files) 2024-11-10T04:23:08,432 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df8778146310ba3408609557bb98b9e5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:08,432 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/194e74659be94a69b5209d4c9fdba1b7, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/0aea2e62f1464e168ad1afbe0840d72c] into tmpdir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp, totalSize=29.3 K 2024-11-10T04:23:08,433 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a4824dc206e471ba64a11383a1d4c39, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731212579059 2024-11-10T04:23:08,433 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting 194e74659be94a69b5209d4c9fdba1b7, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731212586569 2024-11-10T04:23:08,434 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aea2e62f1464e168ad1afbe0840d72c, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731212587990 2024-11-10T04:23:08,453 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df8778146310ba3408609557bb98b9e5#info#compaction#24 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:23:08,453 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/fb4774c507bb4bd8b7f4f90019b6e91a is 1080, key is row0002/info:/1731212579059/Put/seqid=0 2024-11-10T04:23:08,455 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,455 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:08,455 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741885_1068 2024-11-10T04:23:08,456 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:08,459 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37841 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,459 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55502 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741886_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741886_1069 to mirror 127.0.0.1:37841 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:08,459 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:08,459 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741886_1069 2024-11-10T04:23:08,459 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55502 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741886_1069] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:08,459 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:55502 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741886_1069] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55502 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:08,459 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:08,461 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,461 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]) is bad. 2024-11-10T04:23:08,461 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741887_1070 2024-11-10T04:23:08,461 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41831,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK] 2024-11-10T04:23:08,463 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:08,463 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:08,463 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741888_1071 2024-11-10T04:23:08,463 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:08,464 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T04:23:08,464 WARN [IPC Server handler 4 on default port 40839 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T04:23:08,464 WARN [IPC Server handler 4 on default port 40839 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T04:23:08,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741889_1072 (size=18097) 2024-11-10T04:23:08,879 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/fb4774c507bb4bd8b7f4f90019b6e91a as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a 2024-11-10T04:23:08,887 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df8778146310ba3408609557bb98b9e5/info of df8778146310ba3408609557bb98b9e5 into fb4774c507bb4bd8b7f4f90019b6e91a(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:23:08,887 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:08,888 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5., storeName=df8778146310ba3408609557bb98b9e5/info, priority=13, startTime=1731212588430; duration=0sec 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a because midkey is the same as first or last row 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a because midkey is the same as first or last row 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-10T04:23:08,888 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:08,889 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a because midkey is the same as first or last row 2024-11-10T04:23:08,889 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:23:08,889 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df8778146310ba3408609557bb98b9e5:info 2024-11-10T04:23:08,900 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@54803134[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741869_1052 to 127.0.0.1:41831 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:08,900 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@30fd4369[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741874_1057 to 127.0.0.1:37841 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:09,122 WARN [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-11-10T04:23:09,122 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:09,127 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:09,227 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:09,232 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:09,233 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:09,233 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:09,233 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:23:09,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc86a20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:09,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70a14b1c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:09,395 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a8f7772{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/java.io.tmpdir/jetty-localhost-43485-hadoop-hdfs-3_4_1-tests_jar-_-any-5797729254974033083/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:09,395 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d90ffb5{HTTP/1.1, (http/1.1)}{localhost:43485} 2024-11-10T04:23:09,395 INFO [Time-limited test {}] server.Server(415): Started @128940ms 2024-11-10T04:23:09,397 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:23:09,533 WARN [Thread-991 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:23:09,543 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9ba0f60937658d8b with lease ID 0x88ed51acd8d0e3a5: from storage DS-2b0ba88d-61e6-4117-8f04-1db5508cb920 node DatanodeRegistration(127.0.0.1:45839, datanodeUuid=36bda998-80e7-46b9-9d7b-4fc68015f300, infoPort=41019, infoSecurePort=0, ipcPort=41809, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T04:23:09,544 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9ba0f60937658d8b with lease ID 0x88ed51acd8d0e3a5: from storage DS-a4cf91ae-f721-45d3-b122-1b6158c324d0 node DatanodeRegistration(127.0.0.1:45839, datanodeUuid=36bda998-80e7-46b9-9d7b-4fc68015f300, infoPort=41019, infoSecurePort=0, ipcPort=41809, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:09,901 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@54803134[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741884_1067 to 127.0.0.1:37841 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:09,901 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@30fd4369[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741859_1042 to 127.0.0.1:37841 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:10,076 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:11,123 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:11,127 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:11,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741889_1072 (size=18097) 2024-11-10T04:23:12,077 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:13,123 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:13,127 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:14,077 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:14,957 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:23:15,124 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,128 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,234 ERROR [FSHLog-0-hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData-prefix:d1d9e3766cb1,40031,1731212564976 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,234 WARN [FSHLog-0-hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData-prefix:d1d9e3766cb1,40031,1731212564976 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,234 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C40031%2C1731212564976:(num 1731212565146) roll requested 2024-11-10T04:23:15,235 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C40031%2C1731212564976.1731212595234 2024-11-10T04:23:15,239 WARN [Thread-1012 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1073 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40843 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,239 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1617658036_22 at /127.0.0.1:34498 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741890_1073] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4]'}, localName='127.0.0.1:45839', datanodeUuid='36bda998-80e7-46b9-9d7b-4fc68015f300', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741890_1073 to mirror 127.0.0.1:40843 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:15,239 WARN [Thread-1012 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741890_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:15,239 WARN [Thread-1012 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741890_1073 2024-11-10T04:23:15,240 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1617658036_22 at /127.0.0.1:34498 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741890_1073] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:15,240 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1617658036_22 at /127.0.0.1:34498 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741890_1073] {}] datanode.DataXceiver(331): 127.0.0.1:45839:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34498 dst: /127.0.0.1:45839 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:15,240 WARN [Thread-1012 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:15,241 WARN [Thread-1012 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1074 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,242 WARN [Thread-1012 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741891_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:15,242 WARN [Thread-1012 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741891_1074 2024-11-10T04:23:15,242 WARN [Thread-1012 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:15,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:15,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:15,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:15,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:15,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:15,247 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212595234 2024-11-10T04:23:15,248 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,248 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:15,248 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 2024-11-10T04:23:15,248 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41019:41019),(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T04:23:15,248 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 is not closed yet, will try archiving it next time 2024-11-10T04:23:15,248 WARN [IPC Server handler 0 on default port 40839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 has not been closed. Lease recovery is in progress. RecoveryId = 1076 for block blk_1073741830_1006 2024-11-10T04:23:15,249 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 after 1ms 2024-11-10T04:23:16,077 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:17,124 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:18,078 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:19,125 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:19,250 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 after 4002ms 2024-11-10T04:23:19,568 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@7ac6e4c5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:44537,null,null]) java.net.ConnectException: Call From d1d9e3766cb1/172.17.0.2 to localhost:45065 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T04:23:19,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741833_1021 (size=455) 2024-11-10T04:23:20,071 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs/d1d9e3766cb1%2C38007%2C1731212565036.1731212565453 2024-11-10T04:23:20,073 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212585075 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs/d1d9e3766cb1%2C38007%2C1731212565036.1731212585075 2024-11-10T04:23:20,078 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:20,538 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@57c42d7b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45839, datanodeUuid=36bda998-80e7-46b9-9d7b-4fc68015f300, infoPort=41019, infoSecurePort=0, ipcPort=41809, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741833_1021 to 127.0.0.1:40843 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:21,125 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,079 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,739 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.1731212602739 2024-11-10T04:23:22,742 WARN [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,742 WARN [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:22,742 WARN [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741893_1077 2024-11-10T04:23:22,743 WARN [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:22,744 WARN [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,744 WARN [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:22,744 WARN [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741894_1078 2024-11-10T04:23:22,745 WARN [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:22,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:22,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:22,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:22,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:22,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:22,750 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212587106 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212602739 2024-11-10T04:23:22,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41019:41019),(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T04:23:22,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212587106 is not closed yet, will try archiving it next time 2024-11-10T04:23:22,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741879_1062 (size=12911) 2024-11-10T04:23:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38007 {}] regionserver.HRegion(8855): Flush requested on df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:22,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df8778146310ba3408609557bb98b9e5 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T04:23:22,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/6da85f227fd94d7589679b00e7f27828 is 1080, key is row0013/info:/1731212602752/Put/seqid=0 2024-11-10T04:23:22,762 WARN [Thread-1030 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1080 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,762 WARN [Thread-1030 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741896_1080 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:22,762 WARN [Thread-1030 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741896_1080 2024-11-10T04:23:22,763 WARN [Thread-1030 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:22,765 WARN [Thread-1030 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40843 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,765 WARN [Thread-1030 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:22,765 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43546 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741897_1081] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741897_1081 to mirror 127.0.0.1:40843 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:22,765 WARN [Thread-1030 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741897_1081 2024-11-10T04:23:22,765 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43546 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741897_1081] {}] datanode.BlockReceiver(316): Block 1073741897 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:22,765 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43546 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741897_1081] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43546 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:22,766 WARN [Thread-1030 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:22,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741898_1082 (size=8190) 2024-11-10T04:23:22,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741898_1082 (size=8190) 2024-11-10T04:23:22,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/6da85f227fd94d7589679b00e7f27828 2024-11-10T04:23:22,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/6da85f227fd94d7589679b00e7f27828 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/6da85f227fd94d7589679b00e7f27828 2024-11-10T04:23:22,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/6da85f227fd94d7589679b00e7f27828, entries=3, sequenceid=66, filesize=8.0 K 2024-11-10T04:23:22,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for df8778146310ba3408609557bb98b9e5 in 33ms, sequenceid=66, compaction requested=false 2024-11-10T04:23:22,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:22,789 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K 2024-11-10T04:23:22,789 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:22,789 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a because midkey is the same as first or last row 2024-11-10T04:23:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38007 {}] regionserver.HRegion(8855): Flush requested on df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:22,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df8778146310ba3408609557bb98b9e5 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB 2024-11-10T04:23:22,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/d8566db7790943a5b2118b33014c1c8f is 1080, key is row0015/info:/1731212602756/Put/seqid=0 2024-11-10T04:23:22,981 WARN [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1083 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,981 WARN [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741899_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:22,981 WARN [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741899_1083 2024-11-10T04:23:22,982 WARN [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:22,983 WARN [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1084 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,983 WARN [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741900_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:22,983 WARN [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741900_1084 2024-11-10T04:23:22,984 WARN [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:22,986 WARN [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1085 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44537 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:22,986 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43562 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741901_1085] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741901_1085 to mirror 127.0.0.1:44537 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:22,986 WARN [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741901_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:22,986 WARN [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741901_1085 2024-11-10T04:23:22,986 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43562 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741901_1085] {}] datanode.BlockReceiver(316): Block 1073741901 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:22,986 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43562 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741901_1085] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43562 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:22,986 WARN [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:22,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741902_1086 (size=14660) 2024-11-10T04:23:22,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741902_1086 (size=14660) 2024-11-10T04:23:22,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/d8566db7790943a5b2118b33014c1c8f 2024-11-10T04:23:22,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/d8566db7790943a5b2118b33014c1c8f as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/d8566db7790943a5b2118b33014c1c8f 2024-11-10T04:23:23,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/d8566db7790943a5b2118b33014c1c8f, entries=9, sequenceid=79, filesize=14.3 K 2024-11-10T04:23:23,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for df8778146310ba3408609557bb98b9e5 in 31ms, sequenceid=79, compaction requested=true 2024-11-10T04:23:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K 2024-11-10T04:23:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a because midkey is the same as first or last row 2024-11-10T04:23:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df8778146310ba3408609557bb98b9e5:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:23:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:23:23,005 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:23:23,007 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:23:23,007 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1541): df8778146310ba3408609557bb98b9e5/info is initiating minor compaction (all files) 2024-11-10T04:23:23,007 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df8778146310ba3408609557bb98b9e5/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:23,007 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/6da85f227fd94d7589679b00e7f27828, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/d8566db7790943a5b2118b33014c1c8f] into tmpdir=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp, totalSize=40.0 K 2024-11-10T04:23:23,007 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb4774c507bb4bd8b7f4f90019b6e91a, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731212579059 2024-11-10T04:23:23,008 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da85f227fd94d7589679b00e7f27828, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731212589010 2024-11-10T04:23:23,008 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8566db7790943a5b2118b33014c1c8f, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731212602756 2024-11-10T04:23:23,023 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df8778146310ba3408609557bb98b9e5#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:23:23,024 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/91b1ad2f7b264725b8fb0c3c2672d474 is 1080, key is row0002/info:/1731212579059/Put/seqid=0 2024-11-10T04:23:23,026 WARN [Thread-1047 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1087 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,026 WARN [Thread-1047 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741903_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK], DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:23,027 WARN [Thread-1047 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741903_1087 2024-11-10T04:23:23,027 WARN [Thread-1047 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:23,028 WARN [Thread-1047 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1088 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,028 WARN [Thread-1047 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741904_1088 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:23,028 WARN [Thread-1047 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741904_1088 2024-11-10T04:23:23,029 WARN [Thread-1047 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:23,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741905_1089 (size=28989) 2024-11-10T04:23:23,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741905_1089 (size=28989) 2024-11-10T04:23:23,041 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/.tmp/info/91b1ad2f7b264725b8fb0c3c2672d474 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/91b1ad2f7b264725b8fb0c3c2672d474 2024-11-10T04:23:23,049 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df8778146310ba3408609557bb98b9e5/info of df8778146310ba3408609557bb98b9e5 into 91b1ad2f7b264725b8fb0c3c2672d474(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df8778146310ba3408609557bb98b9e5: 2024-11-10T04:23:23,049 INFO [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5., storeName=df8778146310ba3408609557bb98b9e5/info, priority=13, startTime=1731212603005; duration=0sec 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/91b1ad2f7b264725b8fb0c3c2672d474 because midkey is the same as first or last row 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/91b1ad2f7b264725b8fb0c3c2672d474 because midkey is the same as first or last row 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-10T04:23:23,049 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:23:23,050 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/91b1ad2f7b264725b8fb0c3c2672d474 because midkey is the same as first or last row 2024-11-10T04:23:23,050 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:23:23,050 DEBUG [RS:0;d1d9e3766cb1:38007-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df8778146310ba3408609557bb98b9e5:info 2024-11-10T04:23:23,125 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-11-10T04:23:23,125 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,152 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.1731212587106 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs/d1d9e3766cb1%2C38007%2C1731212565036.1731212587106 2024-11-10T04:23:23,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:23:23,175 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:23:23,175 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:23:23,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:23,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:23,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:23:23,175 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:23:23,175 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1750200013, stopped=false 2024-11-10T04:23:23,176 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,40031,1731212564976 2024-11-10T04:23:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:23:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:23:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:23:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:23,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:23:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:23,178 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:23:23,178 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:23:23,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:23,178 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,38007,1731212565036' ***** 2024-11-10T04:23:23,178 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:23:23,178 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,33499,1731212565994' ***** 2024-11-10T04:23:23,178 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:23:23,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:23:23,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:23:23,179 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:23:23,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:23:23,179 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:23:23,179 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:23:23,179 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:23:23,179 INFO [RS:1;d1d9e3766cb1:33499 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:23:23,179 INFO [RS:1;d1d9e3766cb1:33499 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:23:23,179 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,33499,1731212565994 2024-11-10T04:23:23,179 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:23:23,179 INFO [RS:1;d1d9e3766cb1:33499 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d1d9e3766cb1:33499. 2024-11-10T04:23:23,179 DEBUG [RS:1;d1d9e3766cb1:33499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:23:23,179 DEBUG [RS:1;d1d9e3766cb1:33499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:23,179 INFO [RS:0;d1d9e3766cb1:38007 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:23:23,179 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,33499,1731212565994; all regions closed. 2024-11-10T04:23:23,179 INFO [RS:0;d1d9e3766cb1:38007 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(3091): Received CLOSE for df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,38007,1731212565036 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:38007. 2024-11-10T04:23:23,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,180 DEBUG [RS:0;d1d9e3766cb1:38007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:23:23,180 DEBUG [RS:0;d1d9e3766cb1:38007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:23,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:23:23,180 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df8778146310ba3408609557bb98b9e5, disabling compactions & flushes 2024-11-10T04:23:23,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,180 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:23:23,180 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:23,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,180 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:23,181 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. after waiting 0 ms 2024-11-10T04:23:23,181 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:23,181 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T04:23:23,181 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1325): Online Regions={df8778146310ba3408609557bb98b9e5=TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T04:23:23,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,181 DEBUG [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, df8778146310ba3408609557bb98b9e5 2024-11-10T04:23:23,181 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:23:23,181 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:23:23,181 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:23:23,181 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:23:23,181 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/15b104b0aa2346aca82cdfd3557dc209, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/03adef152f8e4efe9214892bd8d376a3, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/194e74659be94a69b5209d4c9fdba1b7, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/0aea2e62f1464e168ad1afbe0840d72c, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/6da85f227fd94d7589679b00e7f27828, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/d8566db7790943a5b2118b33014c1c8f] to archive 2024-11-10T04:23:23,181 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:23:23,181 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,181 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,181 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-10T04:23:23,181 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 2024-11-10T04:23:23,182 ERROR [FSHLog-0-hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6-prefix:d1d9e3766cb1,38007,1731212565036.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,182 WARN [FSHLog-0-hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6-prefix:d1d9e3766cb1,38007,1731212565036.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,182 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38007%2C1731212565036.meta:.meta(num 1731212565845) roll requested 2024-11-10T04:23:23,182 WARN [IPC Server handler 3 on default port 40839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 has not been closed. Lease recovery is in progress. RecoveryId = 1090 for block blk_1073741837_1013 2024-11-10T04:23:23,182 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212603182.meta 2024-11-10T04:23:23,182 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:23:23,183 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 after 2ms 2024-11-10T04:23:23,185 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/15b104b0aa2346aca82cdfd3557dc209 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/15b104b0aa2346aca82cdfd3557dc209 2024-11-10T04:23:23,185 WARN [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741906_1091 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,185 WARN [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741906_1091 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK], DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:23,185 WARN [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741906_1091 2024-11-10T04:23:23,186 WARN [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:23,186 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/abaf90768f9f4daaa780ce350ce0822c 2024-11-10T04:23:23,187 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/5a4824dc206e471ba64a11383a1d4c39 2024-11-10T04:23:23,189 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/03adef152f8e4efe9214892bd8d376a3 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/03adef152f8e4efe9214892bd8d376a3 2024-11-10T04:23:23,189 WARN [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741907_1092 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40843 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,189 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:38992 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741907_1092] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4]'}, localName='127.0.0.1:45839', datanodeUuid='36bda998-80e7-46b9-9d7b-4fc68015f300', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741907_1092 to mirror 127.0.0.1:40843 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,189 WARN [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741907_1092 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:23,189 WARN [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741907_1092 2024-11-10T04:23:23,189 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:38992 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741907_1092] {}] datanode.BlockReceiver(316): Block 1073741907 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:23,189 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:38992 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741907_1092] {}] datanode.DataXceiver(331): 127.0.0.1:45839:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38992 dst: /127.0.0.1:45839 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,189 WARN [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:23,190 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/194e74659be94a69b5209d4c9fdba1b7 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/194e74659be94a69b5209d4c9fdba1b7 2024-11-10T04:23:23,191 WARN [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741908_1093 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44537 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,191 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43582 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741908_1093] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741908_1093 to mirror 127.0.0.1:44537 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,192 WARN [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741908_1093 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:23,192 WARN [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741908_1093 2024-11-10T04:23:23,192 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43582 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741908_1093] {}] datanode.BlockReceiver(316): Block 1073741908 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T04:23:23,192 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/fb4774c507bb4bd8b7f4f90019b6e91a 2024-11-10T04:23:23,192 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43582 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741908_1093] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43582 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,192 WARN [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:23,193 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/0aea2e62f1464e168ad1afbe0840d72c to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/0aea2e62f1464e168ad1afbe0840d72c 2024-11-10T04:23:23,195 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/6da85f227fd94d7589679b00e7f27828 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/6da85f227fd94d7589679b00e7f27828 2024-11-10T04:23:23,196 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/d8566db7790943a5b2118b33014c1c8f to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/info/d8566db7790943a5b2118b33014c1c8f 2024-11-10T04:23:23,196 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d1d9e3766cb1:40031 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T04:23:23,197 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [15b104b0aa2346aca82cdfd3557dc209=10347, abaf90768f9f4daaa780ce350ce0822c=12506, 5a4824dc206e471ba64a11383a1d4c39=17994, 03adef152f8e4efe9214892bd8d376a3=6027, 194e74659be94a69b5209d4c9fdba1b7=6027, fb4774c507bb4bd8b7f4f90019b6e91a=18097, 0aea2e62f1464e168ad1afbe0840d72c=6027, 6da85f227fd94d7589679b00e7f27828=8190, d8566db7790943a5b2118b33014c1c8f=14660] 2024-11-10T04:23:23,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,197 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212603182.meta 2024-11-10T04:23:23,200 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,200 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,201 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta 2024-11-10T04:23:23,201 WARN [IPC Server handler 2 on default port 40839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta has not been closed. Lease recovery is in progress. RecoveryId = 1095 for block blk_1073741834_1010 2024-11-10T04:23:23,201 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta after 0ms 2024-11-10T04:23:23,204 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41019:41019),(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T04:23:23,204 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta is not closed yet, will try archiving it next time 2024-11-10T04:23:23,204 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/default/TestLogRolling-testLogRollOnDatanodeDeath/df8778146310ba3408609557bb98b9e5/recovered.edits/83.seqid, newMaxSeqId=83, maxSeqId=1 2024-11-10T04:23:23,205 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:23,205 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df8778146310ba3408609557bb98b9e5: Waiting for close lock at 1731212603180Running coprocessor pre-close hooks at 1731212603180Disabling compacts and flushes for region at 1731212603180Disabling writes for close at 1731212603181 (+1 ms)Writing region close event to WAL at 1731212603200 (+19 ms)Running coprocessor post-close hooks at 1731212603205 (+5 ms)Closed at 1731212603205 2024-11-10T04:23:23,205 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5. 2024-11-10T04:23:23,220 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/info/6f5a60b77157462dbf8b6daeee5ec0ca is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731212566116.df8778146310ba3408609557bb98b9e5./info:regioninfo/1731212566478/Put/seqid=0 2024-11-10T04:23:23,222 WARN [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741910_1096 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44537 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,222 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43594 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741910_1096] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741910_1096 to mirror 127.0.0.1:44537 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,223 WARN [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741910_1096 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:23,223 WARN [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741910_1096 2024-11-10T04:23:23,223 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43594 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741910_1096] {}] datanode.BlockReceiver(316): Block 1073741910 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:23,223 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43594 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741910_1096] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43594 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,223 WARN [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:23,226 WARN [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741911_1097 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37841 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,226 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39008 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741911_1097] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4]'}, localName='127.0.0.1:45839', datanodeUuid='36bda998-80e7-46b9-9d7b-4fc68015f300', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741911_1097 to mirror 127.0.0.1:37841 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,226 WARN [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741911_1097 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:23,226 WARN [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741911_1097 2024-11-10T04:23:23,226 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39008 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741911_1097] {}] datanode.BlockReceiver(316): Block 1073741911 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:23,226 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39008 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741911_1097] {}] datanode.DataXceiver(331): 127.0.0.1:45839:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39008 dst: /127.0.0.1:45839 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,226 WARN [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:23,228 WARN [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741912_1098 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40843 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,228 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43610 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741912_1098] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741912_1098 to mirror 127.0.0.1:40843 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,228 WARN [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741912_1098 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:23,229 WARN [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741912_1098 2024-11-10T04:23:23,229 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43610 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741912_1098] {}] datanode.BlockReceiver(316): Block 1073741912 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:23,229 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43610 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741912_1098] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43610 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,229 WARN [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:23,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741913_1099 (size=7089) 2024-11-10T04:23:23,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741913_1099 (size=7089) 2024-11-10T04:23:23,237 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/info/6f5a60b77157462dbf8b6daeee5ec0ca 2024-11-10T04:23:23,258 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/ns/90f49e20898443c59402a8697e0f890b is 43, key is default/ns:d/1731212565895/Put/seqid=0 2024-11-10T04:23:23,261 WARN [Thread-1071 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741914_1100 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40843 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,261 WARN [Thread-1071 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741914_1100 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:23,261 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43618 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741914_1100] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741914_1100 to mirror 127.0.0.1:40843 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,261 WARN [Thread-1071 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741914_1100 2024-11-10T04:23:23,261 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43618 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741914_1100] {}] datanode.BlockReceiver(316): Block 1073741914 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:23,261 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43618 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741914_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43618 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,262 WARN [Thread-1071 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:23,264 WARN [Thread-1071 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741915_1101 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37841 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,264 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39036 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741915_1101] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4]'}, localName='127.0.0.1:45839', datanodeUuid='36bda998-80e7-46b9-9d7b-4fc68015f300', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741915_1101 to mirror 127.0.0.1:37841 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,264 WARN [Thread-1071 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741915_1101 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]) is bad. 2024-11-10T04:23:23,264 WARN [Thread-1071 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741915_1101 2024-11-10T04:23:23,264 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39036 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741915_1101] {}] datanode.BlockReceiver(316): Block 1073741915 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:23,264 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39036 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741915_1101] {}] datanode.DataXceiver(331): 127.0.0.1:45839:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39036 dst: /127.0.0.1:45839 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,264 WARN [Thread-1071 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK] 2024-11-10T04:23:23,266 WARN [Thread-1071 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741916_1102 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44537 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,266 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43630 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741916_1102] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10]'}, localName='127.0.0.1:43793', datanodeUuid='364fb77d-14f7-4050-b5bf-d96507181f59', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741916_1102 to mirror 127.0.0.1:44537 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,266 WARN [Thread-1071 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741916_1102 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43793,DS-a1e7c65d-eb7d-415b-90af-c4066ec813ff,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:23,266 WARN [Thread-1071 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741916_1102 2024-11-10T04:23:23,267 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43630 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741916_1102] {}] datanode.BlockReceiver(316): Block 1073741916 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:23,267 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:43630 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741916_1102] {}] datanode.DataXceiver(331): 127.0.0.1:43793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43630 dst: /127.0.0.1:43793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,267 WARN [Thread-1071 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741917_1103 (size=5153) 2024-11-10T04:23:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741917_1103 (size=5153) 2024-11-10T04:23:23,271 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/ns/90f49e20898443c59402a8697e0f890b 2024-11-10T04:23:23,292 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/table/0fdaaf48e7084fa9b110e2e4560011a5 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731212566490/Put/seqid=0 2024-11-10T04:23:23,293 WARN [Thread-1080 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741918_1104 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,294 WARN [Thread-1080 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741918_1104 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK], DatanodeInfoWithStorage[127.0.0.1:37841,DS-2f9ccb04-fcdf-4989-8b01-b125dc5572f0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK]) is bad. 2024-11-10T04:23:23,294 WARN [Thread-1080 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741918_1104 2024-11-10T04:23:23,294 WARN [Thread-1080 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40843,DS-080e8f71-33a6-41d0-a7cb-6135b647691d,DISK] 2024-11-10T04:23:23,296 WARN [Thread-1080 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741919_1105 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44537 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:23,296 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39042 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741919_1105] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4]'}, localName='127.0.0.1:45839', datanodeUuid='36bda998-80e7-46b9-9d7b-4fc68015f300', xmitsInProgress=0}:Exception transferring block BP-1586179451-172.17.0.2-1731212564275:blk_1073741919_1105 to mirror 127.0.0.1:44537 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,296 WARN [Thread-1080 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1586179451-172.17.0.2-1731212564275:blk_1073741919_1105 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45839,DS-2b0ba88d-61e6-4117-8f04-1db5508cb920,DISK], DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK]) is bad. 2024-11-10T04:23:23,296 WARN [Thread-1080 {}] hdfs.DataStreamer(1850): Abandoning BP-1586179451-172.17.0.2-1731212564275:blk_1073741919_1105 2024-11-10T04:23:23,296 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39042 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741919_1105] {}] datanode.BlockReceiver(316): Block 1073741919 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T04:23:23,296 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-839780221_22 at /127.0.0.1:39042 [Receiving block BP-1586179451-172.17.0.2-1731212564275:blk_1073741919_1105] {}] datanode.DataXceiver(331): 127.0.0.1:45839:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39042 dst: /127.0.0.1:45839 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:23,297 WARN [Thread-1080 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44537,DS-895b1448-9869-4c0c-ae27-13bfdf85f0f4,DISK] 2024-11-10T04:23:23,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741920_1106 (size=5424) 2024-11-10T04:23:23,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741920_1106 (size=5424) 2024-11-10T04:23:23,301 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/table/0fdaaf48e7084fa9b110e2e4560011a5 2024-11-10T04:23:23,307 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/info/6f5a60b77157462dbf8b6daeee5ec0ca as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/info/6f5a60b77157462dbf8b6daeee5ec0ca 2024-11-10T04:23:23,308 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:23:23,314 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/info/6f5a60b77157462dbf8b6daeee5ec0ca, entries=10, sequenceid=11, filesize=6.9 K 2024-11-10T04:23:23,315 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/ns/90f49e20898443c59402a8697e0f890b as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/ns/90f49e20898443c59402a8697e0f890b 2024-11-10T04:23:23,320 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/ns/90f49e20898443c59402a8697e0f890b, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T04:23:23,321 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/.tmp/table/0fdaaf48e7084fa9b110e2e4560011a5 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/table/0fdaaf48e7084fa9b110e2e4560011a5 2024-11-10T04:23:23,327 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/table/0fdaaf48e7084fa9b110e2e4560011a5, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T04:23:23,328 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 147ms, sequenceid=11, compaction requested=false 2024-11-10T04:23:23,332 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T04:23:23,333 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:23:23,333 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:23:23,333 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212603181Running coprocessor pre-close hooks at 1731212603181Disabling compacts and flushes for region at 1731212603181Disabling writes for close at 1731212603181Obtaining lock to block concurrent updates at 1731212603181Preparing flush snapshotting stores in 1588230740 at 1731212603181Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731212603182 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731212603205 (+23 ms)Flushing 1588230740/info: creating writer at 1731212603205Flushing 1588230740/info: appending metadata at 1731212603219 (+14 ms)Flushing 1588230740/info: closing flushed file at 1731212603219Flushing 1588230740/ns: creating writer at 1731212603243 (+24 ms)Flushing 1588230740/ns: appending metadata at 1731212603258 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731212603258Flushing 1588230740/table: creating writer at 1731212603277 (+19 ms)Flushing 1588230740/table: appending metadata at 1731212603291 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731212603291Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be09a05: reopening flushed file at 1731212603307 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e9bf3ea: reopening flushed file at 1731212603314 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ab376ba: reopening flushed file at 1731212603321 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 147ms, sequenceid=11, compaction requested=false at 1731212603328 (+7 ms)Writing region close event to WAL at 1731212603329 (+1 ms)Running coprocessor post-close hooks at 1731212603333 (+4 ms)Closed at 1731212603333 2024-11-10T04:23:23,333 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:23:23,381 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,38007,1731212565036; all regions closed. 2024-11-10T04:23:23,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:23,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741909_1094 (size=825) 2024-11-10T04:23:23,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741909_1094 (size=825) 2024-11-10T04:23:23,390 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T04:23:23,390 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T04:23:23,901 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@30fd4369[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43793, datanodeUuid=364fb77d-14f7-4050-b5bf-d96507181f59, infoPort=32943, infoSecurePort=0, ipcPort=42149, storageInfo=lv=-57;cid=testClusterID;nsid=128546038;c=1731212564275):Failed to transfer BP-1586179451-172.17.0.2-1731212564275:blk_1073741879_1062 to 127.0.0.1:40843 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:24,075 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T04:23:24,075 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T04:23:24,077 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:23:25,912 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T04:23:25,912 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T04:23:25,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T04:23:25,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:23:25,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T04:23:26,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:23:26,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741835_1011 (size=393) 2024-11-10T04:23:27,184 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 after 4002ms 2024-11-10T04:23:27,202 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta after 4001ms 2024-11-10T04:23:27,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:23:27,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:23:28,181 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-10T04:23:28,183 DEBUG [RS:1;d1d9e3766cb1:33499 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs 2024-11-10T04:23:28,183 INFO [RS:1;d1d9e3766cb1:33499 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C33499%2C1731212565994:(num 1731212566217) 2024-11-10T04:23:28,184 DEBUG [RS:1;d1d9e3766cb1:33499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:23:28,184 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:23:28,184 INFO [RS:1;d1d9e3766cb1:33499 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33499 2024-11-10T04:23:28,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:23:28,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,33499,1731212565994 2024-11-10T04:23:28,186 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:23:28,188 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,33499,1731212565994] 2024-11-10T04:23:28,189 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,33499,1731212565994 already deleted, retry=false 2024-11-10T04:23:28,189 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,33499,1731212565994 expired; onlineServers=1 2024-11-10T04:23:28,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:28,206 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,220 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,220 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,220 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,221 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,221 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,227 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,228 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:23:28,288 INFO [RS:1;d1d9e3766cb1:33499 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:23:28,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33499-0x101906acf480002, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:23:28,288 INFO [RS:1;d1d9e3766cb1:33499 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,33499,1731212565994; zookeeper connection closed. 2024-11-10T04:23:28,288 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1003422f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1003422f 2024-11-10T04:23:28,382 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-10T04:23:28,386 DEBUG [RS:0;d1d9e3766cb1:38007 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs 2024-11-10T04:23:28,386 INFO [RS:0;d1d9e3766cb1:38007 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38007%2C1731212565036.meta:.meta(num 1731212603182) 2024-11-10T04:23:28,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741895_1079 (size=16308) 2024-11-10T04:23:28,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741895_1079 (size=16308) 2024-11-10T04:23:28,392 DEBUG [RS:0;d1d9e3766cb1:38007 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs 2024-11-10T04:23:28,392 INFO [RS:0;d1d9e3766cb1:38007 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38007%2C1731212565036:(num 1731212602739) 2024-11-10T04:23:28,392 DEBUG [RS:0;d1d9e3766cb1:38007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:28,392 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:23:28,392 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:23:28,392 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:23:28,392 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:23:28,392 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:23:28,392 INFO [RS:0;d1d9e3766cb1:38007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38007 2024-11-10T04:23:28,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,38007,1731212565036 2024-11-10T04:23:28,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:23:28,394 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:23:28,396 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,38007,1731212565036] 2024-11-10T04:23:28,397 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,38007,1731212565036 already deleted, retry=false 2024-11-10T04:23:28,397 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,38007,1731212565036 expired; onlineServers=0 2024-11-10T04:23:28,397 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,40031,1731212564976' ***** 2024-11-10T04:23:28,397 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:23:28,397 INFO [M:0;d1d9e3766cb1:40031 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:23:28,398 INFO [M:0;d1d9e3766cb1:40031 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:23:28,398 DEBUG [M:0;d1d9e3766cb1:40031 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:23:28,398 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:23:28,398 DEBUG [M:0;d1d9e3766cb1:40031 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:23:28,398 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212565236 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212565236,5,FailOnTimeoutGroup] 2024-11-10T04:23:28,398 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212565236 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212565236,5,FailOnTimeoutGroup] 2024-11-10T04:23:28,398 INFO [M:0;d1d9e3766cb1:40031 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:23:28,398 INFO [M:0;d1d9e3766cb1:40031 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:23:28,398 DEBUG [M:0;d1d9e3766cb1:40031 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:23:28,398 INFO [M:0;d1d9e3766cb1:40031 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:23:28,398 INFO [M:0;d1d9e3766cb1:40031 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:23:28,398 INFO [M:0;d1d9e3766cb1:40031 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:23:28,399 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:23:28,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:23:28,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:28,400 DEBUG [M:0;d1d9e3766cb1:40031 {}] zookeeper.ZKUtil(347): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:23:28,400 WARN [M:0;d1d9e3766cb1:40031 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:23:28,400 INFO [M:0;d1d9e3766cb1:40031 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/.lastflushedseqids 2024-11-10T04:23:28,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741921_1107 (size=130) 2024-11-10T04:23:28,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741921_1107 (size=130) 2024-11-10T04:23:28,410 INFO [M:0;d1d9e3766cb1:40031 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:23:28,410 INFO [M:0;d1d9e3766cb1:40031 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:23:28,410 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:23:28,410 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:28,410 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:28,410 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:23:28,410 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:28,410 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB 2024-11-10T04:23:28,427 DEBUG [M:0;d1d9e3766cb1:40031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f878dcabb9e34a3bb2d153a9e11fea50 is 82, key is hbase:meta,,1/info:regioninfo/1731212565877/Put/seqid=0 2024-11-10T04:23:28,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741922_1108 (size=5672) 2024-11-10T04:23:28,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741922_1108 (size=5672) 2024-11-10T04:23:28,432 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f878dcabb9e34a3bb2d153a9e11fea50 2024-11-10T04:23:28,452 DEBUG [M:0;d1d9e3766cb1:40031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3c9f6610818346e88d4b9b52fc50f6b1 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731212566494/Put/seqid=0 2024-11-10T04:23:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741923_1109 (size=6255) 2024-11-10T04:23:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741923_1109 (size=6255) 2024-11-10T04:23:28,458 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3c9f6610818346e88d4b9b52fc50f6b1 2024-11-10T04:23:28,463 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3c9f6610818346e88d4b9b52fc50f6b1 2024-11-10T04:23:28,478 DEBUG [M:0;d1d9e3766cb1:40031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2129b240adf34e10a1edf91aa07f217f is 69, key is d1d9e3766cb1,33499,1731212565994/rs:state/1731212566060/Put/seqid=0 2024-11-10T04:23:28,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741924_1110 (size=5224) 2024-11-10T04:23:28,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741924_1110 (size=5224) 2024-11-10T04:23:28,483 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2129b240adf34e10a1edf91aa07f217f 2024-11-10T04:23:28,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:23:28,496 INFO [RS:0;d1d9e3766cb1:38007 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:23:28,496 INFO [RS:0;d1d9e3766cb1:38007 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,38007,1731212565036; zookeeper connection closed. 2024-11-10T04:23:28,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38007-0x101906acf480001, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:23:28,496 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@378de0ca {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@378de0ca 2024-11-10T04:23:28,496 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-10T04:23:28,502 DEBUG [M:0;d1d9e3766cb1:40031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9e9ccc380ef54a31afe9188bf8f69249 is 52, key is load_balancer_on/state:d/1731212565971/Put/seqid=0 2024-11-10T04:23:28,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741925_1111 (size=5056) 2024-11-10T04:23:28,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741925_1111 (size=5056) 2024-11-10T04:23:28,507 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9e9ccc380ef54a31afe9188bf8f69249 2024-11-10T04:23:28,512 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f878dcabb9e34a3bb2d153a9e11fea50 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f878dcabb9e34a3bb2d153a9e11fea50 2024-11-10T04:23:28,517 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f878dcabb9e34a3bb2d153a9e11fea50, entries=8, sequenceid=60, filesize=5.5 K 2024-11-10T04:23:28,518 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3c9f6610818346e88d4b9b52fc50f6b1 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3c9f6610818346e88d4b9b52fc50f6b1 2024-11-10T04:23:28,523 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3c9f6610818346e88d4b9b52fc50f6b1 2024-11-10T04:23:28,523 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3c9f6610818346e88d4b9b52fc50f6b1, entries=6, sequenceid=60, filesize=6.1 K 2024-11-10T04:23:28,524 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2129b240adf34e10a1edf91aa07f217f as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2129b240adf34e10a1edf91aa07f217f 2024-11-10T04:23:28,528 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2129b240adf34e10a1edf91aa07f217f, entries=2, sequenceid=60, filesize=5.1 K 2024-11-10T04:23:28,529 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9e9ccc380ef54a31afe9188bf8f69249 as hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9e9ccc380ef54a31afe9188bf8f69249 2024-11-10T04:23:28,534 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9e9ccc380ef54a31afe9188bf8f69249, entries=1, sequenceid=60, filesize=4.9 K 2024-11-10T04:23:28,535 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 125ms, sequenceid=60, compaction requested=false 2024-11-10T04:23:28,537 INFO [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:28,537 DEBUG [M:0;d1d9e3766cb1:40031 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212608410Disabling compacts and flushes for region at 1731212608410Disabling writes for close at 1731212608410Obtaining lock to block concurrent updates at 1731212608410Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212608410Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1731212608411 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212608412 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212608412Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212608426 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212608426Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212608438 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212608452 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212608452Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212608463 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212608477 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212608477Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731212608487 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731212608501 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731212608501Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a38702: reopening flushed file at 1731212608511 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@114ded87: reopening flushed file at 1731212608517 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17b20131: reopening flushed file at 1731212608523 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@242ea53a: reopening flushed file at 1731212608529 (+6 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 125ms, sequenceid=60, compaction requested=false at 1731212608535 (+6 ms)Writing region close event to WAL at 1731212608537 (+2 ms)Closed at 1731212608537 2024-11-10T04:23:28,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:28,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741892_1075 (size=1045) 2024-11-10T04:23:28,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45839 is added to blk_1073741892_1075 (size=1045) 2024-11-10T04:23:28,731 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:23:28,754 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,754 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,755 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,755 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,756 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,757 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,763 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:28,767 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:29,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:29,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:29,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:23:29,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43793 is added to blk_1073741836_1012 (size=76) 2024-11-10T04:23:29,572 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2f9e28ea {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:44537,null,null]) java.net.ConnectException: Call From d1d9e3766cb1/172.17.0.2 to localhost:45065 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T04:23:30,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:30,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:30,258 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/WALs/d1d9e3766cb1,40031,1731212564976/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/oldWALs/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 2024-11-10T04:23:30,262 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/MasterData/oldWALs/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146 to hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/oldWALs/d1d9e3766cb1%2C40031%2C1731212564976.1731212565146$masterlocalwal$ 2024-11-10T04:23:30,262 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:23:30,262 INFO [M:0;d1d9e3766cb1:40031 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:23:30,263 INFO [M:0;d1d9e3766cb1:40031 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40031 2024-11-10T04:23:30,263 INFO [M:0;d1d9e3766cb1:40031 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:23:30,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:23:30,365 INFO [M:0;d1d9e3766cb1:40031 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:23:30,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40031-0x101906acf480000, quorum=127.0.0.1:60824, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:23:30,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a8f7772{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:30,368 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d90ffb5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:30,368 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:30,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70a14b1c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:30,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc86a20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:30,370 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:23:30,370 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:23:30,370 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:23:30,370 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1586179451-172.17.0.2-1731212564275 (Datanode Uuid 36bda998-80e7-46b9-9d7b-4fc68015f300) service to localhost/127.0.0.1:40839 2024-11-10T04:23:30,369 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1ae91b8a {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:44537,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:45065 , LocalHost:localPort d1d9e3766cb1/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T04:23:30,370 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1ae91b8a {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:45839,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1586179451-172.17.0.2-1731212564275 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:30,370 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1ae91b8a {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:44537,null,null], DatanodeInfoWithStorage[127.0.0.1:45839,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:44537,null,null], DatanodeInfoWithStorage[127.0.0.1:45839,null,null]] 2024-11-10T04:23:30,370 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1ae91b8a {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:44537,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1586179451-172.17.0.2-1731212564275 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:30,370 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1ae91b8a {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:45839,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1586179451-172.17.0.2-1731212564275 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:30,371 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1ae91b8a {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:44537,null,null], DatanodeInfoWithStorage[127.0.0.1:45839,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1586179451-172.17.0.2-1731212564275:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:44537,null,null], DatanodeInfoWithStorage[127.0.0.1:45839,null,null]] 2024-11-10T04:23:30,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data3/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:30,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data4/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:30,371 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:23:30,373 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c82feab{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:30,374 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74865863{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:30,374 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:30,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f55aa3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:30,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78e445ac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:30,375 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:23:30,375 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:23:30,375 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:23:30,375 WARN [BP-1586179451-172.17.0.2-1731212564275 heartbeating to localhost/127.0.0.1:40839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1586179451-172.17.0.2-1731212564275 (Datanode Uuid 364fb77d-14f7-4050-b5bf-d96507181f59) service to localhost/127.0.0.1:40839 2024-11-10T04:23:30,376 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data9/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:30,376 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/cluster_ff99ee2a-f268-b884-94c7-3cc2a728dda4/data/data10/current/BP-1586179451-172.17.0.2-1731212564275 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:30,376 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:23:30,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cd2a640{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:23:30,383 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64b7b556{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:30,383 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:30,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ed3a961{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:30,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a488aac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:30,391 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:23:30,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:23:30,441 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 78) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40839 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-19-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$897/0x00007f0b58bf4210.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:40839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-18-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-18-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40839 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38185 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40839 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40839 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40839 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40839 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-19-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-19-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:38185 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40839 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$897/0x00007f0b58bf4210.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40839 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-18-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=450 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=121 (was 16) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6583 (was 7623) 2024-11-10T04:23:30,449 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=121, ProcessCount=11, AvailableMemoryMB=6582 2024-11-10T04:23:30,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.log.dir so I do NOT create it in target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f867c6d-3877-7761-9c3f-e901525b29fc/hadoop.tmp.dir so I do NOT create it in target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303, deleteOnExit=true 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/test.cache.data in system properties and HBase conf 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:23:30,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:23:30,451 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:23:30,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:23:30,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:23:30,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:23:30,466 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:23:30,541 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:30,546 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:30,547 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:30,547 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:30,547 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:23:30,548 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:30,548 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1915705e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:30,548 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@347a2271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:30,664 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16304a50{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir/jetty-localhost-38917-hadoop-hdfs-3_4_1-tests_jar-_-any-1473259122687918783/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:23:30,664 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e4fac25{HTTP/1.1, (http/1.1)}{localhost:38917} 2024-11-10T04:23:30,665 INFO [Time-limited test {}] server.Server(415): Started @150210ms 2024-11-10T04:23:30,678 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:23:30,741 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:30,744 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:30,745 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:30,745 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:30,745 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:23:30,746 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc14744{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:30,746 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fefca8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:30,860 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21865735{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir/jetty-localhost-36017-hadoop-hdfs-3_4_1-tests_jar-_-any-7864555397633360718/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:30,860 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54562211{HTTP/1.1, (http/1.1)}{localhost:36017} 2024-11-10T04:23:30,861 INFO [Time-limited test {}] server.Server(415): Started @150406ms 2024-11-10T04:23:30,862 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:23:30,892 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:30,896 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:30,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:30,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:30,897 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:23:30,898 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38cf0a15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:30,898 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a9fc515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:30,962 WARN [Thread-1203 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data1/current/BP-618816429-172.17.0.2-1731212610487/current, will proceed with Du for space computation calculation, 2024-11-10T04:23:30,962 WARN [Thread-1204 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data2/current/BP-618816429-172.17.0.2-1731212610487/current, will proceed with Du for space computation calculation, 2024-11-10T04:23:30,985 WARN [Thread-1182 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:23:30,988 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2e935dc9291283ab with lease ID 0x47c4dd06ad54be62: Processing first storage report for DS-690dea81-623e-4334-8792-f26e6f7011d5 from datanode DatanodeRegistration(127.0.0.1:35301, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=41961, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487) 2024-11-10T04:23:30,988 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2e935dc9291283ab with lease ID 0x47c4dd06ad54be62: from storage DS-690dea81-623e-4334-8792-f26e6f7011d5 node DatanodeRegistration(127.0.0.1:35301, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=41961, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:30,988 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2e935dc9291283ab with lease ID 0x47c4dd06ad54be62: Processing first storage report for DS-f862e8ca-ceb0-48c9-94e9-c5b811bec8c2 from datanode DatanodeRegistration(127.0.0.1:35301, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=41961, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487) 2024-11-10T04:23:30,988 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2e935dc9291283ab with lease ID 0x47c4dd06ad54be62: from storage DS-f862e8ca-ceb0-48c9-94e9-c5b811bec8c2 node DatanodeRegistration(127.0.0.1:35301, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=41961, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T04:23:31,022 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1fc8bed8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir/jetty-localhost-33085-hadoop-hdfs-3_4_1-tests_jar-_-any-11024521632770717886/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:31,022 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44bd8b73{HTTP/1.1, (http/1.1)}{localhost:33085} 2024-11-10T04:23:31,022 INFO [Time-limited test {}] server.Server(415): Started @150567ms 2024-11-10T04:23:31,024 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:23:31,106 WARN [Thread-1229 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data3/current/BP-618816429-172.17.0.2-1731212610487/current, will proceed with Du for space computation calculation, 2024-11-10T04:23:31,106 WARN [Thread-1230 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data4/current/BP-618816429-172.17.0.2-1731212610487/current, will proceed with Du for space computation calculation, 2024-11-10T04:23:31,123 WARN [Thread-1218 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:23:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9491831801516f32 with lease ID 0x47c4dd06ad54be63: Processing first storage report for DS-0da65485-fe1f-43bb-857d-0b56fa73263d from datanode DatanodeRegistration(127.0.0.1:44913, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=39089, infoSecurePort=0, ipcPort=36909, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487) 2024-11-10T04:23:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9491831801516f32 with lease ID 0x47c4dd06ad54be63: from storage DS-0da65485-fe1f-43bb-857d-0b56fa73263d node DatanodeRegistration(127.0.0.1:44913, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=39089, infoSecurePort=0, ipcPort=36909, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9491831801516f32 with lease ID 0x47c4dd06ad54be63: Processing first storage report for DS-c8e55663-b72e-4b63-b16a-acaf2c3242e3 from datanode DatanodeRegistration(127.0.0.1:44913, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=39089, infoSecurePort=0, ipcPort=36909, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487) 2024-11-10T04:23:31,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9491831801516f32 with lease ID 0x47c4dd06ad54be63: from storage DS-c8e55663-b72e-4b63-b16a-acaf2c3242e3 node DatanodeRegistration(127.0.0.1:44913, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=39089, infoSecurePort=0, ipcPort=36909, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:31,148 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc 2024-11-10T04:23:31,151 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/zookeeper_0, clientPort=62550, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:23:31,152 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62550 2024-11-10T04:23:31,152 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:31,153 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:31,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:23:31,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:23:31,164 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514 with version=8 2024-11-10T04:23:31,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase-staging 2024-11-10T04:23:31,166 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:23:31,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:23:31,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:23:31,166 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:23:31,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:23:31,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:23:31,166 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:23:31,166 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:23:31,167 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34613 2024-11-10T04:23:31,168 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34613 connecting to ZooKeeper ensemble=127.0.0.1:62550 2024-11-10T04:23:31,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:346130x0, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:23:31,174 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34613-0x101906b83b60000 connected 2024-11-10T04:23:31,190 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:31,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:31,192 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:31,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:23:31,197 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514, hbase.cluster.distributed=false 2024-11-10T04:23:31,198 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:23:31,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34613 2024-11-10T04:23:31,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34613 2024-11-10T04:23:31,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34613 2024-11-10T04:23:31,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34613 2024-11-10T04:23:31,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34613 2024-11-10T04:23:31,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:31,217 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:23:31,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:23:31,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:23:31,218 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:23:31,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:23:31,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:23:31,218 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:23:31,218 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:23:31,219 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38527 2024-11-10T04:23:31,220 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38527 connecting to ZooKeeper ensemble=127.0.0.1:62550 2024-11-10T04:23:31,221 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:31,222 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:31,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385270x0, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:23:31,227 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385270x0, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:23:31,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38527-0x101906b83b60001 connected 2024-11-10T04:23:31,228 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:23:31,228 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:23:31,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:23:31,230 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:23:31,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38527 2024-11-10T04:23:31,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38527 2024-11-10T04:23:31,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38527 2024-11-10T04:23:31,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38527 2024-11-10T04:23:31,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38527 2024-11-10T04:23:31,259 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:34613 2024-11-10T04:23:31,259 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:31,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:23:31,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:23:31,262 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:31,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:23:31,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,264 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:23:31,265 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,34613,1731212611166 from backup master directory 2024-11-10T04:23:31,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:31,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:23:31,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:23:31,266 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:23:31,266 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:31,271 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/hbase.id] with ID: d15b8417-4135-4a1e-aa78-4fd8dab571de 2024-11-10T04:23:31,271 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/.tmp/hbase.id 2024-11-10T04:23:31,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:23:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:23:31,279 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/.tmp/hbase.id]:[hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/hbase.id] 2024-11-10T04:23:31,291 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:31,292 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:23:31,293 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T04:23:31,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:23:31,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:23:31,303 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:23:31,304 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:23:31,304 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:23:31,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:23:31,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:23:31,714 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store 2024-11-10T04:23:31,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:23:31,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:23:31,723 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:23:31,723 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:23:31,723 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:31,723 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:31,723 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:23:31,723 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:31,723 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:23:31,723 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212611723Disabling compacts and flushes for region at 1731212611723Disabling writes for close at 1731212611723Writing region close event to WAL at 1731212611723Closed at 1731212611723 2024-11-10T04:23:31,724 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/.initializing 2024-11-10T04:23:31,724 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:31,726 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C34613%2C1731212611166, suffix=, logDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166, archiveDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/oldWALs, maxLogs=10 2024-11-10T04:23:31,727 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 2024-11-10T04:23:31,732 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 2024-11-10T04:23:31,734 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39089:39089),(127.0.0.1/127.0.0.1:41961:41961)] 2024-11-10T04:23:31,740 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:23:31,740 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:23:31,740 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,740 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:23:31,743 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:31,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:23:31,745 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:23:31,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:23:31,747 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:23:31,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:23:31,748 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:23:31,749 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,750 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,750 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,751 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,751 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,752 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:23:31,753 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:23:31,756 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:23:31,757 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=721525, jitterRate=-0.08253449201583862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:23:31,757 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212611740Initializing all the Stores at 1731212611741 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212611741Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212611742 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212611742Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212611742Cleaning up temporary data from old regions at 1731212611751 (+9 ms)Region opened successfully at 1731212611757 (+6 ms) 2024-11-10T04:23:31,759 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:23:31,762 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3e48d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:23:31,763 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:23:31,763 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:23:31,763 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:23:31,763 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:23:31,764 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T04:23:31,764 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T04:23:31,764 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:23:31,767 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:23:31,768 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:23:31,769 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:23:31,770 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:23:31,770 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:23:31,772 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:23:31,772 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:23:31,773 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:23:31,774 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:23:31,775 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:23:31,776 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:23:31,778 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:23:31,779 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:23:31,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:23:31,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:23:31,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,782 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,34613,1731212611166, sessionid=0x101906b83b60000, setting cluster-up flag (Was=false) 2024-11-10T04:23:31,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,790 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:23:31,791 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:31,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:31,800 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:23:31,801 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:31,803 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:23:31,804 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:23:31,805 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:23:31,805 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:23:31,805 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,34613,1731212611166 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:23:31,806 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:23:31,807 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:23:31,807 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:23:31,807 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:23:31,807 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:23:31,807 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,807 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:23:31,807 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212641809 2024-11-10T04:23:31,809 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:23:31,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:23:31,809 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:23:31,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:23:31,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:23:31,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:23:31,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:23:31,809 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:23:31,810 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,810 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:23:31,810 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:23:31,810 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:23:31,811 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,811 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:23:31,811 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:23:31,811 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:23:31,811 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212611811,5,FailOnTimeoutGroup] 2024-11-10T04:23:31,811 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212611811,5,FailOnTimeoutGroup] 2024-11-10T04:23:31,811 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,811 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:23:31,811 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,812 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:23:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:23:31,821 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:23:31,821 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514 2024-11-10T04:23:31,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:23:31,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:23:31,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:23:31,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:23:31,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:23:31,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:31,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:23:31,837 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:23:31,837 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:31,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:23:31,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:23:31,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:31,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:23:31,841 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:23:31,841 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:31,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:31,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:23:31,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740 2024-11-10T04:23:31,842 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(746): ClusterId : d15b8417-4135-4a1e-aa78-4fd8dab571de 2024-11-10T04:23:31,843 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:23:31,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740 2024-11-10T04:23:31,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:23:31,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:23:31,845 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:23:31,845 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:23:31,845 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:23:31,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:23:31,847 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:23:31,848 DEBUG [RS:0;d1d9e3766cb1:38527 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f1d2cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:23:31,849 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:23:31,849 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=838600, jitterRate=0.0663357675075531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:23:31,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212611832Initializing all the Stores at 1731212611833 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212611833Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212611833Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212611833Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212611833Cleaning up temporary data from old regions at 1731212611844 (+11 ms)Region opened successfully at 1731212611850 (+6 ms) 2024-11-10T04:23:31,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:23:31,850 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:23:31,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:23:31,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:23:31,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:23:31,851 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:23:31,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212611850Disabling compacts and flushes for region at 1731212611850Disabling writes for close at 1731212611850Writing region close event to WAL at 1731212611851 (+1 ms)Closed at 1731212611851 2024-11-10T04:23:31,852 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:23:31,852 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:23:31,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:23:31,854 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:23:31,855 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:23:31,861 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:38527 2024-11-10T04:23:31,861 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:23:31,861 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:23:31,861 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:23:31,862 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,34613,1731212611166 with port=38527, startcode=1731212611217 2024-11-10T04:23:31,862 DEBUG [RS:0;d1d9e3766cb1:38527 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:23:31,864 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42831, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:23:31,865 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34613 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:31,865 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34613 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:31,867 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514 2024-11-10T04:23:31,867 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40075 2024-11-10T04:23:31,867 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:23:31,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:23:31,869 DEBUG [RS:0;d1d9e3766cb1:38527 {}] zookeeper.ZKUtil(111): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:31,869 WARN [RS:0;d1d9e3766cb1:38527 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:23:31,869 INFO [RS:0;d1d9e3766cb1:38527 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:23:31,870 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:31,870 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,38527,1731212611217] 2024-11-10T04:23:31,873 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:23:31,874 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:23:31,875 INFO [RS:0;d1d9e3766cb1:38527 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:23:31,875 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,876 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:23:31,877 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:23:31,877 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:23:31,877 DEBUG [RS:0;d1d9e3766cb1:38527 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:23:31,878 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,878 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,878 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,878 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,878 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,878 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38527,1731212611217-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:23:31,894 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:23:31,894 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38527,1731212611217-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,894 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,894 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.Replication(171): d1d9e3766cb1,38527,1731212611217 started 2024-11-10T04:23:31,910 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:31,910 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,38527,1731212611217, RpcServer on d1d9e3766cb1/172.17.0.2:38527, sessionid=0x101906b83b60001 2024-11-10T04:23:31,911 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:23:31,911 DEBUG [RS:0;d1d9e3766cb1:38527 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:31,911 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38527,1731212611217' 2024-11-10T04:23:31,911 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:23:31,911 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:23:31,912 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:23:31,912 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:23:31,912 DEBUG [RS:0;d1d9e3766cb1:38527 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:31,912 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38527,1731212611217' 2024-11-10T04:23:31,912 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:23:31,912 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:23:31,913 DEBUG [RS:0;d1d9e3766cb1:38527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:23:31,913 INFO [RS:0;d1d9e3766cb1:38527 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:23:31,913 INFO [RS:0;d1d9e3766cb1:38527 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:23:32,006 WARN [d1d9e3766cb1:34613 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:23:32,015 INFO [RS:0;d1d9e3766cb1:38527 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38527%2C1731212611217, suffix=, logDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217, archiveDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/oldWALs, maxLogs=32 2024-11-10T04:23:32,016 INFO [RS:0;d1d9e3766cb1:38527 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:32,022 INFO [RS:0;d1d9e3766cb1:38527 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:32,023 DEBUG [RS:0;d1d9e3766cb1:38527 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39089:39089),(127.0.0.1/127.0.0.1:41961:41961)] 2024-11-10T04:23:32,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:32,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:32,256 DEBUG [d1d9e3766cb1:34613 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:23:32,256 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:32,258 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38527,1731212611217, state=OPENING 2024-11-10T04:23:32,260 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:23:32,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:32,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:32,262 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:23:32,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:23:32,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:23:32,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38527,1731212611217}] 2024-11-10T04:23:32,416 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:23:32,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:23:32,422 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:23:32,422 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:23:32,424 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38527%2C1731212611217.meta, suffix=.meta, logDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217, archiveDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/oldWALs, maxLogs=32 2024-11-10T04:23:32,424 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta 2024-11-10T04:23:32,430 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta 2024-11-10T04:23:32,431 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41961:41961),(127.0.0.1/127.0.0.1:39089:39089)] 2024-11-10T04:23:32,432 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:23:32,432 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:23:32,432 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:23:32,432 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:23:32,433 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:23:32,433 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:23:32,433 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:23:32,433 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:23:32,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:23:32,435 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:23:32,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:32,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:32,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:23:32,436 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:23:32,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:32,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:32,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:23:32,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:23:32,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:32,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:32,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:23:32,439 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:23:32,439 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:32,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:23:32,439 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:23:32,440 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740 2024-11-10T04:23:32,441 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740 2024-11-10T04:23:32,442 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:23:32,442 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:23:32,443 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:23:32,445 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:23:32,445 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800990, jitterRate=0.018511861562728882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:23:32,445 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:23:32,446 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212612433Writing region info on filesystem at 1731212612433Initializing all the Stores at 1731212612434 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212612434Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212612434Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212612434Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212612434Cleaning up temporary data from old regions at 1731212612442 (+8 ms)Running coprocessor post-open hooks at 1731212612445 (+3 ms)Region opened successfully at 1731212612446 (+1 ms) 2024-11-10T04:23:32,447 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212612416 2024-11-10T04:23:32,450 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:23:32,450 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:23:32,451 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:32,452 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38527,1731212611217, state=OPEN 2024-11-10T04:23:32,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:23:32,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:23:32,457 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:32,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:23:32,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:23:32,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:23:32,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38527,1731212611217 in 194 msec 2024-11-10T04:23:32,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:23:32,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-10T04:23:32,464 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:23:32,464 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:23:32,465 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:23:32,465 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38527,1731212611217, seqNum=-1] 2024-11-10T04:23:32,466 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:23:32,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:23:32,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 667 msec 2024-11-10T04:23:32,472 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212612472, completionTime=-1 2024-11-10T04:23:32,472 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:23:32,472 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:23:32,474 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:23:32,474 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212672474 2024-11-10T04:23:32,474 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731212732474 2024-11-10T04:23:32,474 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-10T04:23:32,475 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,34613,1731212611166-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:32,475 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,34613,1731212611166-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:32,475 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,34613,1731212611166-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:32,475 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:34613, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:32,475 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:32,475 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:32,477 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.213sec 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,34613,1731212611166-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:23:32,479 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,34613,1731212611166-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:23:32,481 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:23:32,482 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:23:32,482 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,34613,1731212611166-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:23:32,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67332887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:23:32,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,34613,-1 for getting cluster id 2024-11-10T04:23:32,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:23:32,545 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15b8417-4135-4a1e-aa78-4fd8dab571de' 2024-11-10T04:23:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:23:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15b8417-4135-4a1e-aa78-4fd8dab571de" 2024-11-10T04:23:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3e09de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:23:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,34613,-1] 2024-11-10T04:23:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:23:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:32,548 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:23:32,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e9dc43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:23:32,549 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:23:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38527,1731212611217, seqNum=-1] 2024-11-10T04:23:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:23:32,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:23:32,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:32,554 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:23:32,556 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T04:23:32,556 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-11-10T04:23:32,557 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-11-10T04:23:32,557 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:23:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1357bafd 2024-11-10T04:23:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:23:32,559 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:23:32,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34613 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T04:23:32,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34613 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T04:23:32,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34613 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:23:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34613 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T04:23:32,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:23:32,563 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:32,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34613 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-11-10T04:23:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:23:32,564 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:23:32,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741835_1011 (size=395) 2024-11-10T04:23:32,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741835_1011 (size=395) 2024-11-10T04:23:32,573 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7360156aa17b3e28b4cdcd29b7862096, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514 2024-11-10T04:23:32,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35301 is added to blk_1073741836_1012 (size=78) 2024-11-10T04:23:32,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44913 is added to blk_1073741836_1012 (size=78) 2024-11-10T04:23:32,580 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:23:32,580 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 7360156aa17b3e28b4cdcd29b7862096, disabling compactions & flushes 2024-11-10T04:23:32,580 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:32,580 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:32,580 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. after waiting 0 ms 2024-11-10T04:23:32,580 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:32,580 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:32,580 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7360156aa17b3e28b4cdcd29b7862096: Waiting for close lock at 1731212612580Disabling compacts and flushes for region at 1731212612580Disabling writes for close at 1731212612580Writing region close event to WAL at 1731212612580Closed at 1731212612580 2024-11-10T04:23:32,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:23:32,582 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731212612582"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212612582"}]},"ts":"1731212612582"} 2024-11-10T04:23:32,585 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:23:32,586 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:23:32,586 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212612586"}]},"ts":"1731212612586"} 2024-11-10T04:23:32,588 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-11-10T04:23:32,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=7360156aa17b3e28b4cdcd29b7862096, ASSIGN}] 2024-11-10T04:23:32,590 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=7360156aa17b3e28b4cdcd29b7862096, ASSIGN 2024-11-10T04:23:32,591 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=7360156aa17b3e28b4cdcd29b7862096, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,38527,1731212611217; forceNewPlan=false, retain=false 2024-11-10T04:23:32,741 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7360156aa17b3e28b4cdcd29b7862096, regionState=OPENING, regionLocation=d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:32,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=7360156aa17b3e28b4cdcd29b7862096, ASSIGN because future has completed 2024-11-10T04:23:32,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7360156aa17b3e28b4cdcd29b7862096, server=d1d9e3766cb1,38527,1731212611217}] 2024-11-10T04:23:32,903 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:32,903 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7360156aa17b3e28b4cdcd29b7862096, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:23:32,903 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,904 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:23:32,904 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,904 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,905 INFO [StoreOpener-7360156aa17b3e28b4cdcd29b7862096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,906 INFO [StoreOpener-7360156aa17b3e28b4cdcd29b7862096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7360156aa17b3e28b4cdcd29b7862096 columnFamilyName info 2024-11-10T04:23:32,906 DEBUG [StoreOpener-7360156aa17b3e28b4cdcd29b7862096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:23:32,907 INFO [StoreOpener-7360156aa17b3e28b4cdcd29b7862096-1 {}] regionserver.HStore(327): Store=7360156aa17b3e28b4cdcd29b7862096/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:23:32,907 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,908 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,908 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,909 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,909 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,911 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,913 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:23:32,913 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7360156aa17b3e28b4cdcd29b7862096; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=708820, jitterRate=-0.0986897200345993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:23:32,913 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:32,914 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7360156aa17b3e28b4cdcd29b7862096: Running coprocessor pre-open hook at 1731212612904Writing region info on filesystem at 1731212612904Initializing all the Stores at 1731212612905 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212612905Cleaning up temporary data from old regions at 1731212612909 (+4 ms)Running coprocessor post-open hooks at 1731212612913 (+4 ms)Region opened successfully at 1731212612914 (+1 ms) 2024-11-10T04:23:32,915 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096., pid=6, masterSystemTime=1731212612898 2024-11-10T04:23:32,918 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:32,918 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:32,919 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7360156aa17b3e28b4cdcd29b7862096, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:32,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7360156aa17b3e28b4cdcd29b7862096, server=d1d9e3766cb1,38527,1731212611217 because future has completed 2024-11-10T04:23:32,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T04:23:32,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7360156aa17b3e28b4cdcd29b7862096, server=d1d9e3766cb1,38527,1731212611217 in 178 msec 2024-11-10T04:23:32,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T04:23:32,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=7360156aa17b3e28b4cdcd29b7862096, ASSIGN in 338 msec 2024-11-10T04:23:32,930 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:23:32,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212612930"}]},"ts":"1731212612930"} 2024-11-10T04:23:32,932 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-11-10T04:23:32,934 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:23:32,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 374 msec 2024-11-10T04:23:33,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:33,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:34,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:34,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:35,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:35,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:35,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T04:23:35,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T04:23:35,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T04:23:35,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-11-10T04:23:35,956 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:23:35,956 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T04:23:36,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:36,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:37,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:37,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:37,935 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:23:37,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,953 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,955 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,956 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,956 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,958 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:23:37,964 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:23:37,965 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-11-10T04:23:38,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:38,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:39,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:39,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:40,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:40,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:41,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:41,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:42,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:42,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:23:42,637 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-11-10T04:23:42,637 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-11-10T04:23:42,640 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T04:23:42,640 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096., hostname=d1d9e3766cb1,38527,1731212611217, seqNum=2] 2024-11-10T04:23:43,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:43,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:44,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:44,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:44,646 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:44,647 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:44,647 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:44,647 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:44913,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:44,647 WARN [DataStreamer for file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 block BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44913,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK], DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44913,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]) is bad. 2024-11-10T04:23:44,647 WARN [DataStreamer for file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta block BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK], DatanodeInfoWithStorage[127.0.0.1:44913,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44913,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]) is bad. 2024-11-10T04:23:44,647 WARN [DataStreamer for file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 block BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44913,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK], DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44913,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]) is bad. 2024-11-10T04:23:44,648 WARN [PacketResponder: BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44913] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:35606 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44913:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35606 dst: /127.0.0.1:44913 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2057429972_22 at /127.0.0.1:35566 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44913:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35566 dst: /127.0.0.1:44913 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:53306 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35301:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53306 dst: /127.0.0.1:35301 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,649 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2057429972_22 at /127.0.0.1:53240 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35301:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53240 dst: /127.0.0.1:35301 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,649 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:35608 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44913:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35608 dst: /127.0.0.1:44913 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,649 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:53280 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35301:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53280 dst: /127.0.0.1:35301 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,652 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1fc8bed8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:44,652 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44bd8b73{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:44,652 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:44,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a9fc515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:44,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38cf0a15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:44,654 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:23:44,654 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:23:44,654 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:23:44,654 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-618816429-172.17.0.2-1731212610487 (Datanode Uuid ff96befa-94ec-4864-89ed-26d96ba3204c) service to localhost/127.0.0.1:40075 2024-11-10T04:23:44,654 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data3/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:44,654 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data4/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:44,655 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:23:44,666 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:44,669 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:44,670 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:44,670 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:44,670 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:23:44,671 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e01093b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:44,671 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b5f8885{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:44,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d648826{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir/jetty-localhost-41857-hadoop-hdfs-3_4_1-tests_jar-_-any-18064136853556884041/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:44,789 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a8d3987{HTTP/1.1, (http/1.1)}{localhost:41857} 2024-11-10T04:23:44,789 INFO [Time-limited test {}] server.Server(415): Started @164334ms 2024-11-10T04:23:44,791 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:23:44,809 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:44,809 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:44,809 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:44,810 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:54586 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35301:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54586 dst: /127.0.0.1:35301 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,810 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2057429972_22 at /127.0.0.1:54600 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35301:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54600 dst: /127.0.0.1:35301 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,810 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:54578 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35301:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54578 dst: /127.0.0.1:35301 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:44,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21865735{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:44,817 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54562211{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:44,817 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:44,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fefca8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:44,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc14744{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:44,819 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:23:44,819 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-618816429-172.17.0.2-1731212610487 (Datanode Uuid d8b0b280-3eaa-4d1f-898f-247129eaea35) service to localhost/127.0.0.1:40075 2024-11-10T04:23:44,819 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:23:44,819 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:23:44,819 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data1/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:44,819 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data2/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:44,820 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:23:44,827 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:44,830 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:44,831 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:44,831 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:44,831 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:23:44,832 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25ea055e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:44,832 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@343c572e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:44,875 WARN [Thread-1353 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:23:44,878 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc75aedfeccb100e with lease ID 0x47c4dd06ad54be64: from storage DS-0da65485-fe1f-43bb-857d-0b56fa73263d node DatanodeRegistration(127.0.0.1:43849, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=42883, infoSecurePort=0, ipcPort=39363, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:44,878 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc75aedfeccb100e with lease ID 0x47c4dd06ad54be64: from storage DS-c8e55663-b72e-4b63-b16a-acaf2c3242e3 node DatanodeRegistration(127.0.0.1:43849, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=42883, infoSecurePort=0, ipcPort=39363, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:44,948 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fd8f940{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir/jetty-localhost-40827-hadoop-hdfs-3_4_1-tests_jar-_-any-5552972900038718188/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:44,949 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7cbeb460{HTTP/1.1, (http/1.1)}{localhost:40827} 2024-11-10T04:23:44,949 INFO [Time-limited test {}] server.Server(415): Started @164494ms 2024-11-10T04:23:44,950 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:23:45,030 WARN [Thread-1384 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:23:45,033 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4003784cf426d44 with lease ID 0x47c4dd06ad54be65: from storage DS-690dea81-623e-4334-8792-f26e6f7011d5 node DatanodeRegistration(127.0.0.1:45317, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=43863, infoSecurePort=0, ipcPort=39439, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:45,033 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4003784cf426d44 with lease ID 0x47c4dd06ad54be65: from storage DS-f862e8ca-ceb0-48c9-94e9-c5b811bec8c2 node DatanodeRegistration(127.0.0.1:45317, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=43863, infoSecurePort=0, ipcPort=39439, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:45,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:45,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:45,968 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-11-10T04:23:45,970 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-11-10T04:23:45,972 ERROR [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:45,972 WARN [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:45,972 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38527%2C1731212611217:(num 1731212612016) roll requested 2024-11-10T04:23:45,972 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:45,978 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 newFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:45,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:45,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:45,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:45,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:45,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:45,979 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:45,979 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:45,979 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:45,979 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:45,980 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43863:43863),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-10T04:23:45,980 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 is not closed yet, will try archiving it next time 2024-11-10T04:23:45,980 WARN [IPC Server handler 0 on default port 40075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013 2024-11-10T04:23:45,980 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 after 1ms 2024-11-10T04:23:46,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:46,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:47,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:47,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:47,983 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-11-10T04:23:48,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:48,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:48,878 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T04:23:49,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:49,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:49,981 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 after 4002ms 2024-11-10T04:23:49,986 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:49,986 WARN [DataStreamer for file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 block BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45317,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK], DatanodeInfoWithStorage[127.0.0.1:43849,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45317,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]) is bad. 2024-11-10T04:23:49,987 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:39020 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:45317:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39020 dst: /127.0.0.1:45317 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:49,987 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:58660 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43849:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58660 dst: /127.0.0.1:43849 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:49,989 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fd8f940{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:49,989 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7cbeb460{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:49,989 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:49,989 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@343c572e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:49,989 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25ea055e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:49,991 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:23:49,991 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:23:49,991 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:23:49,991 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-618816429-172.17.0.2-1731212610487 (Datanode Uuid d8b0b280-3eaa-4d1f-898f-247129eaea35) service to localhost/127.0.0.1:40075 2024-11-10T04:23:49,992 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data1/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:49,992 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data2/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:49,992 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:23:50,006 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:50,010 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:50,011 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:50,011 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:50,011 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:23:50,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45c55ac5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:50,012 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a382d25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:50,131 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@362ea065{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir/jetty-localhost-33723-hadoop-hdfs-3_4_1-tests_jar-_-any-8956167487439689210/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:50,131 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78cc0925{HTTP/1.1, (http/1.1)}{localhost:33723} 2024-11-10T04:23:50,131 INFO [Time-limited test {}] server.Server(415): Started @169677ms 2024-11-10T04:23:50,133 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:23:50,150 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:50,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_381347882_22 at /127.0.0.1:46502 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43849:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46502 dst: /127.0.0.1:43849 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:50,155 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d648826{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:50,156 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a8d3987{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:23:50,156 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:23:50,156 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b5f8885{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:23:50,156 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e01093b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:23:50,157 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:23:50,157 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:23:50,157 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-618816429-172.17.0.2-1731212610487 (Datanode Uuid ff96befa-94ec-4864-89ed-26d96ba3204c) service to localhost/127.0.0.1:40075 2024-11-10T04:23:50,157 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:23:50,158 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data3/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:50,158 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data4/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:23:50,158 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:23:50,172 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:23:50,175 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:23:50,176 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:23:50,176 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:23:50,176 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:23:50,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39c5f69b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:23:50,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5091fc79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:23:50,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:50,232 WARN [Thread-1427 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:23:50,235 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x880a03c5e4542c1d with lease ID 0x47c4dd06ad54be66: from storage DS-690dea81-623e-4334-8792-f26e6f7011d5 node DatanodeRegistration(127.0.0.1:34123, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=38669, infoSecurePort=0, ipcPort=43745, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:50,235 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x880a03c5e4542c1d with lease ID 0x47c4dd06ad54be66: from storage DS-f862e8ca-ceb0-48c9-94e9-c5b811bec8c2 node DatanodeRegistration(127.0.0.1:34123, datanodeUuid=d8b0b280-3eaa-4d1f-898f-247129eaea35, infoPort=38669, infoSecurePort=0, ipcPort=43745, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:50,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:50,293 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@262fed34{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/java.io.tmpdir/jetty-localhost-43911-hadoop-hdfs-3_4_1-tests_jar-_-any-12703497347463512895/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:23:50,294 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f7ae33f{HTTP/1.1, (http/1.1)}{localhost:43911} 2024-11-10T04:23:50,294 INFO [Time-limited test {}] server.Server(415): Started @169839ms 2024-11-10T04:23:50,295 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:23:50,378 WARN [Thread-1458 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:23:50,380 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaef6e5fd87afe2b1 with lease ID 0x47c4dd06ad54be67: from storage DS-0da65485-fe1f-43bb-857d-0b56fa73263d node DatanodeRegistration(127.0.0.1:33007, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=41915, infoSecurePort=0, ipcPort=35849, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:50,381 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaef6e5fd87afe2b1 with lease ID 0x47c4dd06ad54be67: from storage DS-c8e55663-b72e-4b63-b16a-acaf2c3242e3 node DatanodeRegistration(127.0.0.1:33007, datanodeUuid=ff96befa-94ec-4864-89ed-26d96ba3204c, infoPort=41915, infoSecurePort=0, ipcPort=35849, storageInfo=lv=-57;cid=testClusterID;nsid=243636326;c=1731212610487), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:23:51,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:51,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:51,313 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-11-10T04:23:51,315 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-11-10T04:23:51,317 ERROR [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43849,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:51,317 WARN [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43849,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:51,317 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38527%2C1731212611217:(num 1731212625972) roll requested 2024-11-10T04:23:51,317 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 2024-11-10T04:23:51,323 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 newFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 2024-11-10T04:23:51,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:51,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:51,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:51,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:51,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:51,324 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 2024-11-10T04:23:51,324 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43849,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:51,324 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43849,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:51,324 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:51,324 WARN [IPC Server handler 2 on default port 40075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-11-10T04:23:51,324 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 after 0ms 2024-11-10T04:23:51,328 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:41915:41915)] 2024-11-10T04:23:51,328 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 is not closed yet, will try archiving it next time 2024-11-10T04:23:52,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:52,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:53,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:53,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:53,330 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:53,335 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 newFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:53,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:53,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:53,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:53,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:53,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:53,336 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:53,337 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41915:41915),(127.0.0.1/127.0.0.1:38669:38669)] 2024-11-10T04:23:53,337 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 is not closed yet, will try archiving it next time 2024-11-10T04:23:53,337 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 is not closed yet, will try archiving it next time 2024-11-10T04:23:53,338 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:53,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:53,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741838_1019 (size=1264) 2024-11-10T04:23:53,339 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 after 0ms 2024-11-10T04:23:53,339 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:53,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741838_1019 (size=1264) 2024-11-10T04:23:53,339 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 is not closed yet, will try archiving it next time 2024-11-10T04:23:53,348 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731212612914/Put/vlen=218/seqid=0] 2024-11-10T04:23:53,348 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731212622645/Put/vlen=1045/seqid=0] 2024-11-10T04:23:53,348 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212612016 2024-11-10T04:23:53,348 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:53,348 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:53,349 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 after 1ms 2024-11-10T04:23:53,349 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:53,352 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731212625971/Put/vlen=1045/seqid=0] 2024-11-10T04:23:53,352 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731212627984/Put/vlen=1045/seqid=0] 2024-11-10T04:23:53,352 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 2024-11-10T04:23:53,352 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 2024-11-10T04:23:53,352 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 2024-11-10T04:23:53,352 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 after 0ms 2024-11-10T04:23:53,352 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212631317 2024-11-10T04:23:53,355 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731212631317/Put/vlen=1045/seqid=0] 2024-11-10T04:23:53,355 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:53,355 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:53,356 WARN [IPC Server handler 1 on default port 40075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-11-10T04:23:53,356 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 after 1ms 2024-11-10T04:23:54,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:54,239 WARN [ResponseProcessor for block BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:54,239 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2057429972_22 at /127.0.0.1:35148 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:33007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35148 dst: /127.0.0.1:33007 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33007 remote=/127.0.0.1:35148]. Total timeout mills is 60000, 59096 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:54,240 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2057429972_22 at /127.0.0.1:40004 [Receiving block BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:34123:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40004 dst: /127.0.0.1:34123 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:23:54,240 WARN [DataStreamer for file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 block BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33007,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK], DatanodeInfoWithStorage[127.0.0.1:34123,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33007,DS-0da65485-fe1f-43bb-857d-0b56fa73263d,DISK]) is bad. 2024-11-10T04:23:54,241 WARN [DataStreamer for file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 block BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:54,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741839_1022 (size=85) 2024-11-10T04:23:54,246 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:55,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:55,235 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T04:23:55,246 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:55,325 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212625972 after 4001ms 2024-11-10T04:23:56,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:56,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:57,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:57,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:57,357 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 after 4001ms 2024-11-10T04:23:57,357 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:57,360 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:57,361 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7360156aa17b3e28b4cdcd29b7862096 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-11-10T04:23:57,361 ERROR [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,361 WARN [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,362 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38527%2C1731212611217:(num 1731212633330) roll requested 2024-11-10T04:23:57,362 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38527%2C1731212611217.1731212637362 2024-11-10T04:23:57,367 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 newFile=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212637362 2024-11-10T04:23:57,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,368 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212637362 2024-11-10T04:23:57,368 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,369 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-618816429-172.17.0.2-1731212610487:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,369 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:57,369 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 after 0ms 2024-11-10T04:23:57,370 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:41915:41915)] 2024-11-10T04:23:57,370 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 to hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/oldWALs/d1d9e3766cb1%2C38527%2C1731212611217.1731212633330 2024-11-10T04:23:57,385 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096/.tmp/info/816ac1b1536e42d0812ee6c7ef5889bf is 1080, key is row1002/info:/1731212622645/Put/seqid=0 2024-11-10T04:23:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741841_1024 (size=9270) 2024-11-10T04:23:57,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741841_1024 (size=9270) 2024-11-10T04:23:57,391 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096/.tmp/info/816ac1b1536e42d0812ee6c7ef5889bf 2024-11-10T04:23:57,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096/.tmp/info/816ac1b1536e42d0812ee6c7ef5889bf as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096/info/816ac1b1536e42d0812ee6c7ef5889bf 2024-11-10T04:23:57,404 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096/info/816ac1b1536e42d0812ee6c7ef5889bf, entries=4, sequenceid=8, filesize=9.1 K 2024-11-10T04:23:57,405 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 7360156aa17b3e28b4cdcd29b7862096 in 45ms, sequenceid=8, compaction requested=false 2024-11-10T04:23:57,405 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7360156aa17b3e28b4cdcd29b7862096: 2024-11-10T04:23:57,406 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-11-10T04:23:57,406 ERROR [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,406 WARN [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514-prefix:d1d9e3766cb1,38527,1731212611217.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,406 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C38527%2C1731212611217.meta:.meta(num 1731212612424) roll requested 2024-11-10T04:23:57,406 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212637406.meta 2024-11-10T04:23:57,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,412 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212637406.meta 2024-11-10T04:23:57,412 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,412 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:23:57,412 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta 2024-11-10T04:23:57,413 WARN [IPC Server handler 3 on default port 40075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014 2024-11-10T04:23:57,413 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta after 1ms 2024-11-10T04:23:57,416 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:41915:41915)] 2024-11-10T04:23:57,416 DEBUG [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta is not closed yet, will try archiving it next time 2024-11-10T04:23:57,431 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/info/1e9c0d72d0cf4e89b96eb37c8cdd9375 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096./info:regioninfo/1731212612919/Put/seqid=0 2024-11-10T04:23:57,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741843_1027 (size=7125) 2024-11-10T04:23:57,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741843_1027 (size=7125) 2024-11-10T04:23:57,437 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/info/1e9c0d72d0cf4e89b96eb37c8cdd9375 2024-11-10T04:23:57,457 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/ns/df59c8f361c2485bac147cc00ee50f0c is 43, key is default/ns:d/1731212612467/Put/seqid=0 2024-11-10T04:23:57,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741844_1028 (size=5153) 2024-11-10T04:23:57,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741844_1028 (size=5153) 2024-11-10T04:23:57,462 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/ns/df59c8f361c2485bac147cc00ee50f0c 2024-11-10T04:23:57,482 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/table/8b71a463ec934c6ebbddc3bc4eebb6e8 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731212612930/Put/seqid=0 2024-11-10T04:23:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741845_1029 (size=5438) 2024-11-10T04:23:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741845_1029 (size=5438) 2024-11-10T04:23:57,487 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/table/8b71a463ec934c6ebbddc3bc4eebb6e8 2024-11-10T04:23:57,493 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/info/1e9c0d72d0cf4e89b96eb37c8cdd9375 as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/info/1e9c0d72d0cf4e89b96eb37c8cdd9375 2024-11-10T04:23:57,498 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/info/1e9c0d72d0cf4e89b96eb37c8cdd9375, entries=10, sequenceid=11, filesize=7.0 K 2024-11-10T04:23:57,499 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/ns/df59c8f361c2485bac147cc00ee50f0c as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/ns/df59c8f361c2485bac147cc00ee50f0c 2024-11-10T04:23:57,505 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/ns/df59c8f361c2485bac147cc00ee50f0c, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T04:23:57,506 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/.tmp/table/8b71a463ec934c6ebbddc3bc4eebb6e8 as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/table/8b71a463ec934c6ebbddc3bc4eebb6e8 2024-11-10T04:23:57,512 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/table/8b71a463ec934c6ebbddc3bc4eebb6e8, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T04:23:57,513 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=11, compaction requested=false 2024-11-10T04:23:57,514 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T04:23:57,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:23:57,520 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:23:57,520 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:23:57,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:57,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:23:57,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:57,521 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:23:57,521 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=102419092, stopped=false 2024-11-10T04:23:57,521 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,34613,1731212611166 2024-11-10T04:23:57,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:23:57,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:23:57,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:57,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:23:57,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:23:57,524 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:23:57,524 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:23:57,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:57,524 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,38527,1731212611217' ***** 2024-11-10T04:23:57,524 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:23:57,525 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:23:57,525 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:23:57,525 INFO [RS:0;d1d9e3766cb1:38527 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:23:57,525 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:23:57,525 INFO [RS:0;d1d9e3766cb1:38527 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:23:57,525 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(3091): Received CLOSE for 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:57,525 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,38527,1731212611217 2024-11-10T04:23:57,525 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:23:57,525 INFO [RS:0;d1d9e3766cb1:38527 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:38527. 2024-11-10T04:23:57,526 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:23:57,526 DEBUG [RS:0;d1d9e3766cb1:38527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:23:57,526 DEBUG [RS:0;d1d9e3766cb1:38527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:23:57,526 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7360156aa17b3e28b4cdcd29b7862096, disabling compactions & flushes 2024-11-10T04:23:57,526 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:23:57,526 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:57,526 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:23:57,526 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:23:57,526 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:57,526 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. after waiting 0 ms 2024-11-10T04:23:57,526 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:23:57,526 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:57,526 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T04:23:57,526 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1325): Online Regions={7360156aa17b3e28b4cdcd29b7862096=TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T04:23:57,526 DEBUG [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7360156aa17b3e28b4cdcd29b7862096 2024-11-10T04:23:57,526 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:23:57,527 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:23:57,527 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:23:57,527 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:23:57,527 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:23:57,531 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/default/TestLogRolling-testLogRollOnPipelineRestart/7360156aa17b3e28b4cdcd29b7862096/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-10T04:23:57,532 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:57,532 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T04:23:57,532 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7360156aa17b3e28b4cdcd29b7862096: Waiting for close lock at 1731212637526Running coprocessor pre-close hooks at 1731212637526Disabling compacts and flushes for region at 1731212637526Disabling writes for close at 1731212637526Writing region close event to WAL at 1731212637527 (+1 ms)Running coprocessor post-close hooks at 1731212637532 (+5 ms)Closed at 1731212637532 2024-11-10T04:23:57,532 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731212612560.7360156aa17b3e28b4cdcd29b7862096. 2024-11-10T04:23:57,532 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:23:57,532 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:23:57,532 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212637526Running coprocessor pre-close hooks at 1731212637526Disabling compacts and flushes for region at 1731212637526Disabling writes for close at 1731212637527 (+1 ms)Writing region close event to WAL at 1731212637528 (+1 ms)Running coprocessor post-close hooks at 1731212637532 (+4 ms)Closed at 1731212637532 2024-11-10T04:23:57,533 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:23:57,727 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,38527,1731212611217; all regions closed. 2024-11-10T04:23:57,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:23:57,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741842_1025 (size=825) 2024-11-10T04:23:57,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741842_1025 (size=825) 2024-11-10T04:23:57,879 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:23:57,909 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T04:23:57,909 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T04:23:58,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:58,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:59,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:23:59,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:00,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:00,249 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:01,148 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:24:01,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:01,249 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:01,382 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T04:24:01,414 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta after 4002ms 2024-11-10T04:24:01,414 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/WALs/d1d9e3766cb1,38527,1731212611217/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta to hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/oldWALs/d1d9e3766cb1%2C38527%2C1731212611217.meta.1731212612424.meta 2024-11-10T04:24:01,417 DEBUG [RS:0;d1d9e3766cb1:38527 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/oldWALs 2024-11-10T04:24:01,417 INFO [RS:0;d1d9e3766cb1:38527 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38527%2C1731212611217.meta:.meta(num 1731212637406) 2024-11-10T04:24:01,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741840_1023 (size=1162) 2024-11-10T04:24:01,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741840_1023 (size=1162) 2024-11-10T04:24:01,425 DEBUG [RS:0;d1d9e3766cb1:38527 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/oldWALs 2024-11-10T04:24:01,425 INFO [RS:0;d1d9e3766cb1:38527 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38527%2C1731212611217:(num 1731212637362) 2024-11-10T04:24:01,425 DEBUG [RS:0;d1d9e3766cb1:38527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:01,425 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:24:01,425 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:24:01,425 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:24:01,425 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:24:01,425 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:24:01,425 INFO [RS:0;d1d9e3766cb1:38527 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38527 2024-11-10T04:24:01,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:24:01,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,38527,1731212611217 2024-11-10T04:24:01,428 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:24:01,431 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,38527,1731212611217] 2024-11-10T04:24:01,433 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,38527,1731212611217 already deleted, retry=false 2024-11-10T04:24:01,433 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,38527,1731212611217 expired; onlineServers=0 2024-11-10T04:24:01,433 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,34613,1731212611166' ***** 2024-11-10T04:24:01,433 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:24:01,433 INFO [M:0;d1d9e3766cb1:34613 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:24:01,433 INFO [M:0;d1d9e3766cb1:34613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:24:01,434 DEBUG [M:0;d1d9e3766cb1:34613 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:24:01,434 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:24:01,434 DEBUG [M:0;d1d9e3766cb1:34613 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:24:01,434 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212611811 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212611811,5,FailOnTimeoutGroup] 2024-11-10T04:24:01,434 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212611811 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212611811,5,FailOnTimeoutGroup] 2024-11-10T04:24:01,434 INFO [M:0;d1d9e3766cb1:34613 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:24:01,434 INFO [M:0;d1d9e3766cb1:34613 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:24:01,434 DEBUG [M:0;d1d9e3766cb1:34613 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:24:01,434 INFO [M:0;d1d9e3766cb1:34613 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:24:01,434 INFO [M:0;d1d9e3766cb1:34613 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:24:01,435 INFO [M:0;d1d9e3766cb1:34613 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:24:01,435 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:24:01,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:24:01,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:01,436 DEBUG [M:0;d1d9e3766cb1:34613 {}] zookeeper.ZKUtil(347): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:24:01,436 WARN [M:0;d1d9e3766cb1:34613 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:24:01,436 INFO [M:0;d1d9e3766cb1:34613 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/.lastflushedseqids 2024-11-10T04:24:01,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741846_1030 (size=130) 2024-11-10T04:24:01,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741846_1030 (size=130) 2024-11-10T04:24:01,442 INFO [M:0;d1d9e3766cb1:34613 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:24:01,442 INFO [M:0;d1d9e3766cb1:34613 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:24:01,442 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:24:01,442 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:01,443 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:01,443 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:24:01,443 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:01,443 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB 2024-11-10T04:24:01,443 ERROR [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData-prefix:d1d9e3766cb1,34613,1731212611166 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:24:01,443 WARN [FSHLog-0-hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData-prefix:d1d9e3766cb1,34613,1731212611166 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:24:01,443 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d1d9e3766cb1%2C34613%2C1731212611166:(num 1731212611727) roll requested 2024-11-10T04:24:01,443 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C34613%2C1731212611166.1731212641443 2024-11-10T04:24:01,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,448 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212641443 2024-11-10T04:24:01,449 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:24:01,449 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35301,DS-690dea81-623e-4334-8792-f26e6f7011d5,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T04:24:01,449 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 2024-11-10T04:24:01,449 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41915:41915),(127.0.0.1/127.0.0.1:38669:38669)] 2024-11-10T04:24:01,449 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 is not closed yet, will try archiving it next time 2024-11-10T04:24:01,449 WARN [IPC Server handler 0 on default port 40075 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015 2024-11-10T04:24:01,450 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 after 1ms 2024-11-10T04:24:01,471 DEBUG [M:0;d1d9e3766cb1:34613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5441a6bd06094de2b226c303d0fbc1a7 is 82, key is hbase:meta,,1/info:regioninfo/1731212612451/Put/seqid=0 2024-11-10T04:24:01,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741848_1033 (size=5672) 2024-11-10T04:24:01,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741848_1033 (size=5672) 2024-11-10T04:24:01,476 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5441a6bd06094de2b226c303d0fbc1a7 2024-11-10T04:24:01,496 DEBUG [M:0;d1d9e3766cb1:34613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96661afd907142f5aae140601ed170d0 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731212612935/Put/seqid=0 2024-11-10T04:24:01,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741849_1034 (size=6119) 2024-11-10T04:24:01,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741849_1034 (size=6119) 2024-11-10T04:24:01,501 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96661afd907142f5aae140601ed170d0 2024-11-10T04:24:01,522 DEBUG [M:0;d1d9e3766cb1:34613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d035104d5f0a4faf89b850af696085da is 69, key is d1d9e3766cb1,38527,1731212611217/rs:state/1731212611865/Put/seqid=0 2024-11-10T04:24:01,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741850_1035 (size=5156) 2024-11-10T04:24:01,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741850_1035 (size=5156) 2024-11-10T04:24:01,527 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d035104d5f0a4faf89b850af696085da 2024-11-10T04:24:01,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:01,531 INFO [RS:0;d1d9e3766cb1:38527 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:24:01,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38527-0x101906b83b60001, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:01,531 INFO [RS:0;d1d9e3766cb1:38527 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,38527,1731212611217; zookeeper connection closed. 2024-11-10T04:24:01,531 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4cc9af3d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4cc9af3d 2024-11-10T04:24:01,532 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:24:01,546 DEBUG [M:0;d1d9e3766cb1:34613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/03f23cbaf95541a2a11cdd2da10a3cc3 is 52, key is load_balancer_on/state:d/1731212612555/Put/seqid=0 2024-11-10T04:24:01,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741851_1036 (size=5056) 2024-11-10T04:24:01,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741851_1036 (size=5056) 2024-11-10T04:24:01,551 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/03f23cbaf95541a2a11cdd2da10a3cc3 2024-11-10T04:24:01,558 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5441a6bd06094de2b226c303d0fbc1a7 as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5441a6bd06094de2b226c303d0fbc1a7 2024-11-10T04:24:01,563 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5441a6bd06094de2b226c303d0fbc1a7, entries=8, sequenceid=56, filesize=5.5 K 2024-11-10T04:24:01,564 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96661afd907142f5aae140601ed170d0 as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96661afd907142f5aae140601ed170d0 2024-11-10T04:24:01,570 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96661afd907142f5aae140601ed170d0, entries=6, sequenceid=56, filesize=6.0 K 2024-11-10T04:24:01,570 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d035104d5f0a4faf89b850af696085da as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d035104d5f0a4faf89b850af696085da 2024-11-10T04:24:01,575 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d035104d5f0a4faf89b850af696085da, entries=1, sequenceid=56, filesize=5.0 K 2024-11-10T04:24:01,576 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/03f23cbaf95541a2a11cdd2da10a3cc3 as hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/03f23cbaf95541a2a11cdd2da10a3cc3 2024-11-10T04:24:01,581 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/03f23cbaf95541a2a11cdd2da10a3cc3, entries=1, sequenceid=56, filesize=4.9 K 2024-11-10T04:24:01,583 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=56, compaction requested=false 2024-11-10T04:24:01,584 INFO [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:01,584 DEBUG [M:0;d1d9e3766cb1:34613 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212641442Disabling compacts and flushes for region at 1731212641442Disabling writes for close at 1731212641443 (+1 ms)Obtaining lock to block concurrent updates at 1731212641443Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212641443Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731212641443Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212641450 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212641450Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212641471 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212641471Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212641482 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212641495 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212641496 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212641507 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212641522 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212641522Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731212641532 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731212641545 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731212641545Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5447a2dd: reopening flushed file at 1731212641557 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2c7834: reopening flushed file at 1731212641563 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558a4577: reopening flushed file at 1731212641570 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c5c7533: reopening flushed file at 1731212641575 (+5 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=56, compaction requested=false at 1731212641583 (+8 ms)Writing region close event to WAL at 1731212641584 (+1 ms)Closed at 1731212641584 2024-11-10T04:24:01,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34123 is added to blk_1073741847_1031 (size=757) 2024-11-10T04:24:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33007 is added to blk_1073741847_1031 (size=757) 2024-11-10T04:24:02,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:02,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:02,532 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,532 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,546 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,546 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,550 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,550 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,550 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,552 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,556 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:02,556 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,058 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:24:03,059 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,060 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,060 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,060 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,076 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,077 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,077 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,077 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,077 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,077 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,080 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,081 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,081 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,082 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:03,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:03,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:04,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:04,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:04,381 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T04:24:05,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:05,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:05,450 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 after 4001ms 2024-11-10T04:24:05,451 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/WALs/d1d9e3766cb1,34613,1731212611166/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 to hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/oldWALs/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 2024-11-10T04:24:05,453 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/MasterData/oldWALs/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727 to hdfs://localhost:40075/user/jenkins/test-data/3b2eacea-3a38-7875-b148-f7d477a2b514/oldWALs/d1d9e3766cb1%2C34613%2C1731212611166.1731212611727$masterlocalwal$ 2024-11-10T04:24:05,454 INFO [M:0;d1d9e3766cb1:34613 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:24:05,454 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:24:05,454 INFO [M:0;d1d9e3766cb1:34613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34613 2024-11-10T04:24:05,454 INFO [M:0;d1d9e3766cb1:34613 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:24:05,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:05,558 INFO [M:0;d1d9e3766cb1:34613 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:24:05,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34613-0x101906b83b60000, quorum=127.0.0.1:62550, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:05,560 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@262fed34{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:05,561 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f7ae33f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:24:05,561 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:24:05,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5091fc79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:24:05,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39c5f69b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:24:05,562 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:24:05,562 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:24:05,562 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:24:05,562 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-618816429-172.17.0.2-1731212610487 (Datanode Uuid ff96befa-94ec-4864-89ed-26d96ba3204c) service to localhost/127.0.0.1:40075 2024-11-10T04:24:05,563 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data3/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:05,563 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data4/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:05,563 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:24:05,565 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@362ea065{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:05,565 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78cc0925{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:24:05,565 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:24:05,565 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a382d25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:24:05,565 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45c55ac5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:24:05,567 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:24:05,567 WARN [BP-618816429-172.17.0.2-1731212610487 heartbeating to localhost/127.0.0.1:40075 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-618816429-172.17.0.2-1731212610487 (Datanode Uuid d8b0b280-3eaa-4d1f-898f-247129eaea35) service to localhost/127.0.0.1:40075 2024-11-10T04:24:05,567 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:24:05,567 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:24:05,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data1/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:05,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/cluster_b5c485c8-105f-e019-9906-fdff773c8303/data/data2/current/BP-618816429-172.17.0.2-1731212610487 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:05,568 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:24:05,573 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16304a50{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:24:05,573 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e4fac25{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:24:05,573 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:24:05,573 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@347a2271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:24:05,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1915705e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:24:05,579 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:24:05,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:24:05,604 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40075 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40075 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:40075 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40075 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40075 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40075 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40075 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40075 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 121), ProcessCount=11 (was 11), AvailableMemoryMB=6429 (was 6582) 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=6429 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.log.dir so I do NOT create it in target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7eae17f2-f323-bc5d-a17f-eb34eaced5cc/hadoop.tmp.dir so I do NOT create it in target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887, deleteOnExit=true 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/test.cache.data in system properties and HBase conf 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:24:05,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:24:05,613 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:24:05,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:24:05,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:24:05,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:24:05,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:24:05,627 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:24:05,691 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:05,696 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:24:05,697 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:24:05,697 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:24:05,697 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:24:05,698 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:05,698 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3cb009a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:24:05,698 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dd613b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:24:05,812 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c447ff3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/java.io.tmpdir/jetty-localhost-34077-hadoop-hdfs-3_4_1-tests_jar-_-any-17349912378679909687/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:24:05,813 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@301adf20{HTTP/1.1, (http/1.1)}{localhost:34077} 2024-11-10T04:24:05,813 INFO [Time-limited test {}] server.Server(415): Started @185358ms 2024-11-10T04:24:05,825 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:24:05,874 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:05,877 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:24:05,878 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:24:05,878 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:24:05,878 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:24:05,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3df88721{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:24:05,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f08894b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:24:05,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:24:05,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T04:24:05,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T04:24:05,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T04:24:05,993 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67c2b9b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/java.io.tmpdir/jetty-localhost-36957-hadoop-hdfs-3_4_1-tests_jar-_-any-13114668616187393413/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:05,994 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6dc3ea71{HTTP/1.1, (http/1.1)}{localhost:36957} 2024-11-10T04:24:05,994 INFO [Time-limited test {}] server.Server(415): Started @185539ms 2024-11-10T04:24:05,995 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:24:06,022 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:06,025 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:24:06,026 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:24:06,026 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:24:06,026 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:24:06,026 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35023063{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:24:06,027 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8afa355{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:24:06,079 WARN [Thread-1652 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data1/current/BP-295375615-172.17.0.2-1731212645642/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:06,079 WARN [Thread-1653 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data2/current/BP-295375615-172.17.0.2-1731212645642/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:06,094 WARN [Thread-1631 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:24:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b74e7d111f8b188 with lease ID 0xb704ba606a3f5bbb: Processing first storage report for DS-ca633288-a34b-4782-8527-c380b5532bae from datanode DatanodeRegistration(127.0.0.1:38023, datanodeUuid=f27a0ae3-ef4d-4e66-af48-4ac02730ba0c, infoPort=38631, infoSecurePort=0, ipcPort=38575, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642) 2024-11-10T04:24:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b74e7d111f8b188 with lease ID 0xb704ba606a3f5bbb: from storage DS-ca633288-a34b-4782-8527-c380b5532bae node DatanodeRegistration(127.0.0.1:38023, datanodeUuid=f27a0ae3-ef4d-4e66-af48-4ac02730ba0c, infoPort=38631, infoSecurePort=0, ipcPort=38575, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b74e7d111f8b188 with lease ID 0xb704ba606a3f5bbb: Processing first storage report for DS-82356b2a-750d-4409-a216-45d7bd5e4caa from datanode DatanodeRegistration(127.0.0.1:38023, datanodeUuid=f27a0ae3-ef4d-4e66-af48-4ac02730ba0c, infoPort=38631, infoSecurePort=0, ipcPort=38575, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642) 2024-11-10T04:24:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b74e7d111f8b188 with lease ID 0xb704ba606a3f5bbb: from storage DS-82356b2a-750d-4409-a216-45d7bd5e4caa node DatanodeRegistration(127.0.0.1:38023, datanodeUuid=f27a0ae3-ef4d-4e66-af48-4ac02730ba0c, infoPort=38631, infoSecurePort=0, ipcPort=38575, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:06,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6007e850{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/java.io.tmpdir/jetty-localhost-46861-hadoop-hdfs-3_4_1-tests_jar-_-any-7870990479332541913/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:06,144 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@18701e65{HTTP/1.1, (http/1.1)}{localhost:46861} 2024-11-10T04:24:06,144 INFO [Time-limited test {}] server.Server(415): Started @185690ms 2024-11-10T04:24:06,146 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:24:06,215 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:06,230 WARN [Thread-1678 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data3/current/BP-295375615-172.17.0.2-1731212645642/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:06,230 WARN [Thread-1679 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data4/current/BP-295375615-172.17.0.2-1731212645642/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:06,246 WARN [Thread-1667 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:24:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc76a1e65ac5e3743 with lease ID 0xb704ba606a3f5bbc: Processing first storage report for DS-845ee56b-d3a9-4df3-b1a4-c87754e425ef from datanode DatanodeRegistration(127.0.0.1:39271, datanodeUuid=b358d3e2-e6ce-412b-b611-85fbdffc1833, infoPort=36307, infoSecurePort=0, ipcPort=40519, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642) 2024-11-10T04:24:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc76a1e65ac5e3743 with lease ID 0xb704ba606a3f5bbc: from storage DS-845ee56b-d3a9-4df3-b1a4-c87754e425ef node DatanodeRegistration(127.0.0.1:39271, datanodeUuid=b358d3e2-e6ce-412b-b611-85fbdffc1833, infoPort=36307, infoSecurePort=0, ipcPort=40519, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc76a1e65ac5e3743 with lease ID 0xb704ba606a3f5bbc: Processing first storage report for DS-52f42c29-2c95-48cd-9e19-67a1330140e7 from datanode DatanodeRegistration(127.0.0.1:39271, datanodeUuid=b358d3e2-e6ce-412b-b611-85fbdffc1833, infoPort=36307, infoSecurePort=0, ipcPort=40519, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642) 2024-11-10T04:24:06,249 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc76a1e65ac5e3743 with lease ID 0xb704ba606a3f5bbc: from storage DS-52f42c29-2c95-48cd-9e19-67a1330140e7 node DatanodeRegistration(127.0.0.1:39271, datanodeUuid=b358d3e2-e6ce-412b-b611-85fbdffc1833, infoPort=36307, infoSecurePort=0, ipcPort=40519, storageInfo=lv=-57;cid=testClusterID;nsid=183101650;c=1731212645642), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:06,252 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:06,268 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc 2024-11-10T04:24:06,270 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/zookeeper_0, clientPort=56992, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:24:06,271 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56992 2024-11-10T04:24:06,271 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:06,273 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:06,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:24:06,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:24:06,283 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772 with version=8 2024-11-10T04:24:06,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase-staging 2024-11-10T04:24:06,286 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:24:06,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:06,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:06,286 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:24:06,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:06,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:24:06,286 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:24:06,286 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:24:06,287 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43815 2024-11-10T04:24:06,289 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43815 connecting to ZooKeeper ensemble=127.0.0.1:56992 2024-11-10T04:24:06,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:438150x0, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:24:06,297 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43815-0x101906c0ce60000 connected 2024-11-10T04:24:06,320 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:06,322 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:06,323 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:24:06,324 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772, hbase.cluster.distributed=false 2024-11-10T04:24:06,325 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:24:06,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43815 2024-11-10T04:24:06,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43815 2024-11-10T04:24:06,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43815 2024-11-10T04:24:06,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43815 2024-11-10T04:24:06,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43815 2024-11-10T04:24:06,342 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:24:06,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:06,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:06,342 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:24:06,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:06,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:24:06,342 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:24:06,343 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:24:06,343 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38659 2024-11-10T04:24:06,344 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38659 connecting to ZooKeeper ensemble=127.0.0.1:56992 2024-11-10T04:24:06,345 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:06,347 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:06,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386590x0, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:24:06,351 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38659-0x101906c0ce60001 connected 2024-11-10T04:24:06,351 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:24:06,352 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:24:06,354 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:24:06,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:24:06,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:24:06,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38659 2024-11-10T04:24:06,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38659 2024-11-10T04:24:06,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38659 2024-11-10T04:24:06,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38659 2024-11-10T04:24:06,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38659 2024-11-10T04:24:06,373 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:43815 2024-11-10T04:24:06,374 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:06,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:06,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:06,376 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:06,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:24:06,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,379 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:24:06,379 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,43815,1731212646285 from backup master directory 2024-11-10T04:24:06,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:06,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:06,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:06,381 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:24:06,381 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:06,385 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/hbase.id] with ID: d8797bbd-c988-40e1-890a-606b4e81db77 2024-11-10T04:24:06,385 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/.tmp/hbase.id 2024-11-10T04:24:06,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:24:06,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:24:06,392 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/.tmp/hbase.id]:[hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/hbase.id] 2024-11-10T04:24:06,402 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:06,402 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:24:06,404 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-10T04:24:06,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:24:06,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:24:06,413 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:24:06,414 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:24:06,414 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:24:06,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:24:06,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:24:06,422 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store 2024-11-10T04:24:06,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:24:06,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:24:06,428 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:06,428 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:24:06,428 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:06,428 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:06,428 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:24:06,428 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:06,428 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:06,428 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212646428Disabling compacts and flushes for region at 1731212646428Disabling writes for close at 1731212646428Writing region close event to WAL at 1731212646428Closed at 1731212646428 2024-11-10T04:24:06,429 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/.initializing 2024-11-10T04:24:06,429 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/WALs/d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:06,431 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C43815%2C1731212646285, suffix=, logDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/WALs/d1d9e3766cb1,43815,1731212646285, archiveDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/oldWALs, maxLogs=10 2024-11-10T04:24:06,432 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C43815%2C1731212646285.1731212646432 2024-11-10T04:24:06,441 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/WALs/d1d9e3766cb1,43815,1731212646285/d1d9e3766cb1%2C43815%2C1731212646285.1731212646432 2024-11-10T04:24:06,442 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36307:36307),(127.0.0.1/127.0.0.1:38631:38631)] 2024-11-10T04:24:06,442 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:24:06,442 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:06,442 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,442 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:24:06,445 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:06,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:24:06,447 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:06,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:24:06,448 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:06,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:24:06,450 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:06,450 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,451 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,451 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,452 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,452 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,453 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:24:06,454 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:06,456 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:24:06,456 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=749618, jitterRate=-0.0468115508556366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:24:06,457 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212646443Initializing all the Stores at 1731212646443Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212646443Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212646443Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212646444 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212646444Cleaning up temporary data from old regions at 1731212646452 (+8 ms)Region opened successfully at 1731212646457 (+5 ms) 2024-11-10T04:24:06,457 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:24:06,460 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41af9dbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:24:06,461 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:24:06,461 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:24:06,461 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:24:06,461 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:24:06,461 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T04:24:06,462 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T04:24:06,462 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:24:06,464 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:24:06,464 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:24:06,466 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:24:06,466 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:24:06,467 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:24:06,468 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:24:06,468 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:24:06,469 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:24:06,471 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:24:06,472 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:24:06,473 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:24:06,474 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:24:06,476 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:24:06,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:24:06,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:24:06,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,478 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,43815,1731212646285, sessionid=0x101906c0ce60000, setting cluster-up flag (Was=false) 2024-11-10T04:24:06,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,487 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:24:06,488 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:06,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,498 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:24:06,498 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:06,499 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:24:06,501 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:06,501 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:24:06,501 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:24:06,502 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,43815,1731212646285 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:24:06,503 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212676504 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,504 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:24:06,505 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:24:06,505 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:06,505 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:24:06,505 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:24:06,505 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:24:06,505 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:24:06,505 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212646505,5,FailOnTimeoutGroup] 2024-11-10T04:24:06,505 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212646505,5,FailOnTimeoutGroup] 2024-11-10T04:24:06,505 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,506 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:24:06,506 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,506 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,506 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,506 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:24:06,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:24:06,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:24:06,513 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:24:06,513 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772 2024-11-10T04:24:06,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:24:06,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:24:06,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:06,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:24:06,522 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:24:06,522 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:06,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:24:06,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:24:06,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:06,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:24:06,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:24:06,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:06,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:24:06,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:24:06,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:06,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:06,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:24:06,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740 2024-11-10T04:24:06,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740 2024-11-10T04:24:06,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:24:06,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:24:06,530 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:24:06,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:24:06,532 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:24:06,533 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834856, jitterRate=0.061575502157211304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:24:06,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212646520Initializing all the Stores at 1731212646521 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212646521Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212646521Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212646521Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212646521Cleaning up temporary data from old regions at 1731212646529 (+8 ms)Region opened successfully at 1731212646534 (+5 ms) 2024-11-10T04:24:06,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:24:06,534 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:24:06,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:24:06,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:24:06,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:24:06,534 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:24:06,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212646534Disabling compacts and flushes for region at 1731212646534Disabling writes for close at 1731212646534Writing region close event to WAL at 1731212646534Closed at 1731212646534 2024-11-10T04:24:06,535 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:06,535 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:24:06,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:24:06,537 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:24:06,538 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:24:06,563 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(746): ClusterId : d8797bbd-c988-40e1-890a-606b4e81db77 2024-11-10T04:24:06,563 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:24:06,566 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:24:06,566 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:24:06,570 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:24:06,570 DEBUG [RS:0;d1d9e3766cb1:38659 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6aff89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:24:06,582 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:38659 2024-11-10T04:24:06,582 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:24:06,582 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:24:06,582 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:24:06,583 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,43815,1731212646285 with port=38659, startcode=1731212646342 2024-11-10T04:24:06,583 DEBUG [RS:0;d1d9e3766cb1:38659 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:24:06,585 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43371, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:24:06,585 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43815 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:06,585 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43815 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:06,587 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772 2024-11-10T04:24:06,587 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33403 2024-11-10T04:24:06,587 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:24:06,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:24:06,590 DEBUG [RS:0;d1d9e3766cb1:38659 {}] zookeeper.ZKUtil(111): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:06,590 WARN [RS:0;d1d9e3766cb1:38659 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:24:06,590 INFO [RS:0;d1d9e3766cb1:38659 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:24:06,590 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:06,590 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,38659,1731212646342] 2024-11-10T04:24:06,593 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:24:06,595 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:24:06,595 INFO [RS:0;d1d9e3766cb1:38659 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:24:06,595 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,595 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:24:06,596 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:24:06,596 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,596 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,596 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,596 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,596 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,596 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,596 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:24:06,597 DEBUG [RS:0;d1d9e3766cb1:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:24:06,598 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,598 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,598 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,598 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,598 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,598 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38659,1731212646342-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:24:06,614 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:24:06,614 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38659,1731212646342-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,615 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,615 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.Replication(171): d1d9e3766cb1,38659,1731212646342 started 2024-11-10T04:24:06,630 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:06,630 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,38659,1731212646342, RpcServer on d1d9e3766cb1/172.17.0.2:38659, sessionid=0x101906c0ce60001 2024-11-10T04:24:06,630 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:24:06,630 DEBUG [RS:0;d1d9e3766cb1:38659 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:06,630 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38659,1731212646342' 2024-11-10T04:24:06,631 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:24:06,631 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:24:06,632 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:24:06,632 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:24:06,632 DEBUG [RS:0;d1d9e3766cb1:38659 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:06,632 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38659,1731212646342' 2024-11-10T04:24:06,632 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:24:06,632 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:24:06,633 DEBUG [RS:0;d1d9e3766cb1:38659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:24:06,633 INFO [RS:0;d1d9e3766cb1:38659 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:24:06,633 INFO [RS:0;d1d9e3766cb1:38659 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:24:06,689 WARN [d1d9e3766cb1:43815 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:24:06,735 INFO [RS:0;d1d9e3766cb1:38659 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38659%2C1731212646342, suffix=, logDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342, archiveDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/oldWALs, maxLogs=32 2024-11-10T04:24:06,735 INFO [RS:0;d1d9e3766cb1:38659 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38659%2C1731212646342.1731212646735 2024-11-10T04:24:06,741 INFO [RS:0;d1d9e3766cb1:38659 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212646735 2024-11-10T04:24:06,742 DEBUG [RS:0;d1d9e3766cb1:38659 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38631:38631),(127.0.0.1/127.0.0.1:36307:36307)] 2024-11-10T04:24:06,939 DEBUG [d1d9e3766cb1:43815 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:24:06,939 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:06,941 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38659,1731212646342, state=OPENING 2024-11-10T04:24:06,943 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:24:06,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:06,947 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:24:06,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38659,1731212646342}] 2024-11-10T04:24:06,947 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:06,947 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:07,101 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:24:07,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38157, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:24:07,106 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:24:07,106 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:24:07,108 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38659%2C1731212646342.meta, suffix=.meta, logDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342, archiveDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/oldWALs, maxLogs=32 2024-11-10T04:24:07,108 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38659%2C1731212646342.meta.1731212647108.meta 2024-11-10T04:24:07,113 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.meta.1731212647108.meta 2024-11-10T04:24:07,114 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36307:36307),(127.0.0.1/127.0.0.1:38631:38631)] 2024-11-10T04:24:07,114 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:24:07,115 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:24:07,115 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:24:07,115 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:24:07,115 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:24:07,115 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:07,115 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:24:07,115 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:24:07,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:24:07,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:24:07,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:07,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:07,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:24:07,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:24:07,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:07,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:07,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:24:07,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:24:07,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:07,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:07,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:24:07,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:24:07,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:07,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:07,121 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:24:07,122 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740 2024-11-10T04:24:07,123 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740 2024-11-10T04:24:07,124 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:24:07,124 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:24:07,124 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:24:07,126 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:24:07,126 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804416, jitterRate=0.02286839485168457}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:24:07,126 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:24:07,127 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212647115Writing region info on filesystem at 1731212647115Initializing all the Stores at 1731212647116 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212647116Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212647116Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212647116Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212647116Cleaning up temporary data from old regions at 1731212647124 (+8 ms)Running coprocessor post-open hooks at 1731212647126 (+2 ms)Region opened successfully at 1731212647127 (+1 ms) 2024-11-10T04:24:07,128 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212647100 2024-11-10T04:24:07,130 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:24:07,130 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:24:07,131 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:07,132 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38659,1731212646342, state=OPEN 2024-11-10T04:24:07,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:24:07,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:24:07,137 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:07,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:07,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:07,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:24:07,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38659,1731212646342 in 190 msec 2024-11-10T04:24:07,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:24:07,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 604 msec 2024-11-10T04:24:07,144 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:07,144 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:24:07,145 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:24:07,145 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38659,1731212646342, seqNum=-1] 2024-11-10T04:24:07,146 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:24:07,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54313, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:24:07,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 651 msec 2024-11-10T04:24:07,153 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212647153, completionTime=-1 2024-11-10T04:24:07,153 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:24:07,153 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:24:07,155 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:24:07,155 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212707155 2024-11-10T04:24:07,156 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731212767156 2024-11-10T04:24:07,156 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T04:24:07,156 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43815,1731212646285-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:07,156 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43815,1731212646285-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:07,156 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43815,1731212646285-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:07,156 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:43815, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:07,156 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:07,157 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:07,158 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.779sec 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43815,1731212646285-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:24:07,160 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43815,1731212646285-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:24:07,163 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:24:07,163 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:24:07,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b173219, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:24:07,163 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,43815,1731212646285-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:07,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,43815,-1 for getting cluster id 2024-11-10T04:24:07,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:24:07,165 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd8797bbd-c988-40e1-890a-606b4e81db77' 2024-11-10T04:24:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:24:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d8797bbd-c988-40e1-890a-606b4e81db77" 2024-11-10T04:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23273074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,43815,-1] 2024-11-10T04:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:24:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:07,167 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:24:07,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307cf8f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:24:07,169 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:24:07,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38659,1731212646342, seqNum=-1] 2024-11-10T04:24:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:24:07,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:24:07,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:07,172 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:07,174 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T04:24:07,175 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:24:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a9663e3 2024-11-10T04:24:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:24:07,176 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:24:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T04:24:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T04:24:07,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:24:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:07,180 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:24:07,180 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:07,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-10T04:24:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:24:07,181 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:24:07,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741835_1011 (size=405) 2024-11-10T04:24:07,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741835_1011 (size=405) 2024-11-10T04:24:07,189 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d55d2cfc4b6c9ab5665fa7f430ea318f, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772 2024-11-10T04:24:07,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741836_1012 (size=88) 2024-11-10T04:24:07,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741836_1012 (size=88) 2024-11-10T04:24:07,196 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:07,196 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing d55d2cfc4b6c9ab5665fa7f430ea318f, disabling compactions & flushes 2024-11-10T04:24:07,196 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:07,196 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:07,196 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. after waiting 0 ms 2024-11-10T04:24:07,196 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:07,196 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:07,196 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for d55d2cfc4b6c9ab5665fa7f430ea318f: Waiting for close lock at 1731212647196Disabling compacts and flushes for region at 1731212647196Disabling writes for close at 1731212647196Writing region close event to WAL at 1731212647196Closed at 1731212647196 2024-11-10T04:24:07,198 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:24:07,198 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731212647198"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212647198"}]},"ts":"1731212647198"} 2024-11-10T04:24:07,200 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:24:07,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:24:07,202 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212647201"}]},"ts":"1731212647201"} 2024-11-10T04:24:07,204 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-10T04:24:07,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d55d2cfc4b6c9ab5665fa7f430ea318f, ASSIGN}] 2024-11-10T04:24:07,205 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d55d2cfc4b6c9ab5665fa7f430ea318f, ASSIGN 2024-11-10T04:24:07,206 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d55d2cfc4b6c9ab5665fa7f430ea318f, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,38659,1731212646342; forceNewPlan=false, retain=false 2024-11-10T04:24:07,215 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:07,252 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:07,357 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d55d2cfc4b6c9ab5665fa7f430ea318f, regionState=OPENING, regionLocation=d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:07,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d55d2cfc4b6c9ab5665fa7f430ea318f, ASSIGN because future has completed 2024-11-10T04:24:07,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d55d2cfc4b6c9ab5665fa7f430ea318f, server=d1d9e3766cb1,38659,1731212646342}] 2024-11-10T04:24:07,517 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:07,517 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d55d2cfc4b6c9ab5665fa7f430ea318f, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:24:07,518 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,518 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:07,518 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,518 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,519 INFO [StoreOpener-d55d2cfc4b6c9ab5665fa7f430ea318f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,521 INFO [StoreOpener-d55d2cfc4b6c9ab5665fa7f430ea318f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d55d2cfc4b6c9ab5665fa7f430ea318f columnFamilyName info 2024-11-10T04:24:07,521 DEBUG [StoreOpener-d55d2cfc4b6c9ab5665fa7f430ea318f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:07,521 INFO [StoreOpener-d55d2cfc4b6c9ab5665fa7f430ea318f-1 {}] regionserver.HStore(327): Store=d55d2cfc4b6c9ab5665fa7f430ea318f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:07,521 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,522 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,522 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,523 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,523 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,524 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,526 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:24:07,527 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d55d2cfc4b6c9ab5665fa7f430ea318f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=865484, jitterRate=0.10052068531513214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:24:07,527 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:07,527 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d55d2cfc4b6c9ab5665fa7f430ea318f: Running coprocessor pre-open hook at 1731212647518Writing region info on filesystem at 1731212647518Initializing all the Stores at 1731212647519 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212647519Cleaning up temporary data from old regions at 1731212647523 (+4 ms)Running coprocessor post-open hooks at 1731212647527 (+4 ms)Region opened successfully at 1731212647527 2024-11-10T04:24:07,528 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f., pid=6, masterSystemTime=1731212647513 2024-11-10T04:24:07,531 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:07,531 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:07,531 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d55d2cfc4b6c9ab5665fa7f430ea318f, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:07,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d55d2cfc4b6c9ab5665fa7f430ea318f, server=d1d9e3766cb1,38659,1731212646342 because future has completed 2024-11-10T04:24:07,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T04:24:07,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d55d2cfc4b6c9ab5665fa7f430ea318f, server=d1d9e3766cb1,38659,1731212646342 in 175 msec 2024-11-10T04:24:07,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T04:24:07,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d55d2cfc4b6c9ab5665fa7f430ea318f, ASSIGN in 334 msec 2024-11-10T04:24:07,541 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:24:07,541 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212647541"}]},"ts":"1731212647541"} 2024-11-10T04:24:07,543 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-10T04:24:07,544 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:24:07,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 367 msec 2024-11-10T04:24:08,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:08,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:09,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:09,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:10,217 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:10,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:11,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:11,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:11,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:24:11,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,475 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,475 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,475 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,475 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,476 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,476 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,479 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,479 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,480 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:11,482 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:24:12,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:12,255 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:12,593 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:24:12,594 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-10T04:24:13,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:13,255 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:14,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:14,256 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:15,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:15,256 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:15,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T04:24:15,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T04:24:15,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:24:15,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T04:24:15,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T04:24:15,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T04:24:15,956 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:15,956 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T04:24:16,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:16,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:24:17,217 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T04:24:17,218 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-10T04:24:17,221 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:17,221 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:17,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f., hostname=d1d9e3766cb1,38659,1731212646342, seqNum=2] 2024-11-10T04:24:17,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:17,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:24:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:24:17,239 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:24:17,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:24:17,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:17,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38659 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:24:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:17,402 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing d55d2cfc4b6c9ab5665fa7f430ea318f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T04:24:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/c1bc8769d89f467d8cecd80d629a0271 is 1080, key is row0001/info:/1731212657225/Put/seqid=0 2024-11-10T04:24:17,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741837_1013 (size=6033) 2024-11-10T04:24:17,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741837_1013 (size=6033) 2024-11-10T04:24:17,427 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/c1bc8769d89f467d8cecd80d629a0271 2024-11-10T04:24:17,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/c1bc8769d89f467d8cecd80d629a0271 as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/c1bc8769d89f467d8cecd80d629a0271 2024-11-10T04:24:17,440 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/c1bc8769d89f467d8cecd80d629a0271, entries=1, sequenceid=5, filesize=5.9 K 2024-11-10T04:24:17,441 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d55d2cfc4b6c9ab5665fa7f430ea318f in 39ms, sequenceid=5, compaction requested=false 2024-11-10T04:24:17,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for d55d2cfc4b6c9ab5665fa7f430ea318f: 2024-11-10T04:24:17,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T04:24:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T04:24:17,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T04:24:17,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 206 msec 2024-11-10T04:24:17,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 216 msec 2024-11-10T04:24:18,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:18,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:19,222 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:19,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:20,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:20,259 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:21,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:21,259 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:22,224 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:22,260 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:23,224 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:23,261 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:24,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:24,261 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:25,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:25,262 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:26,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:26,262 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:27,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:27,262 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:24:27,287 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T04:24:27,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:24:27,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:24:27,294 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:24:27,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:24:27,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38659 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:24:27,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:27,448 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing d55d2cfc4b6c9ab5665fa7f430ea318f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T04:24:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/21e18c8ff9b249b1ad3c9852f6745f3e is 1080, key is row0002/info:/1731212667289/Put/seqid=0 2024-11-10T04:24:27,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741838_1014 (size=6033) 2024-11-10T04:24:27,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741838_1014 (size=6033) 2024-11-10T04:24:27,863 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/21e18c8ff9b249b1ad3c9852f6745f3e 2024-11-10T04:24:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/21e18c8ff9b249b1ad3c9852f6745f3e as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/21e18c8ff9b249b1ad3c9852f6745f3e 2024-11-10T04:24:27,875 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/21e18c8ff9b249b1ad3c9852f6745f3e, entries=1, sequenceid=9, filesize=5.9 K 2024-11-10T04:24:27,876 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d55d2cfc4b6c9ab5665fa7f430ea318f in 429ms, sequenceid=9, compaction requested=false 2024-11-10T04:24:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for d55d2cfc4b6c9ab5665fa7f430ea318f: 2024-11-10T04:24:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T04:24:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T04:24:27,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T04:24:27,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 584 msec 2024-11-10T04:24:27,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 590 msec 2024-11-10T04:24:28,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:28,263 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:29,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:29,264 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:30,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:30,264 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:31,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:31,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 after 68048ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:24:31,265 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:31,265 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta after 68064ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:24:32,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:32,265 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:33,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:33,266 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:34,231 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:34,267 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:35,231 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:35,267 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:36,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:36,267 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:24:36,268 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:37,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:37,268 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:24:37,377 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T04:24:37,380 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38659%2C1731212646342.1731212677380 2024-11-10T04:24:37,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:37,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:37,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:37,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:37,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:37,387 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212646735 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212677380 2024-11-10T04:24:37,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741833_1009 (size=5546) 2024-11-10T04:24:37,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741833_1009 (size=5546) 2024-11-10T04:24:37,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38631:38631),(127.0.0.1/127.0.0.1:36307:36307)] 2024-11-10T04:24:37,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:24:37,395 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:24:37,396 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:24:37,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:24:37,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38659 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:24:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:37,550 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing d55d2cfc4b6c9ab5665fa7f430ea318f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T04:24:37,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/4b6e7d7067f745a1bada1976b962dbb6 is 1080, key is row0003/info:/1731212677379/Put/seqid=0 2024-11-10T04:24:37,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741840_1016 (size=6033) 2024-11-10T04:24:37,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741840_1016 (size=6033) 2024-11-10T04:24:37,560 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/4b6e7d7067f745a1bada1976b962dbb6 2024-11-10T04:24:37,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/4b6e7d7067f745a1bada1976b962dbb6 as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4b6e7d7067f745a1bada1976b962dbb6 2024-11-10T04:24:37,572 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4b6e7d7067f745a1bada1976b962dbb6, entries=1, sequenceid=13, filesize=5.9 K 2024-11-10T04:24:37,573 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d55d2cfc4b6c9ab5665fa7f430ea318f in 23ms, sequenceid=13, compaction requested=true 2024-11-10T04:24:37,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for d55d2cfc4b6c9ab5665fa7f430ea318f: 2024-11-10T04:24:37,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:37,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T04:24:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T04:24:37,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T04:24:37,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-11-10T04:24:37,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec 2024-11-10T04:24:38,233 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:38,269 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:39,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:39,269 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:40,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:40,269 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:41,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:41,270 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:42,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:42,270 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:43,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:43,271 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:44,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:44,271 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:45,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:45,272 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:46,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:46,272 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:47,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:47,273 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:47,370 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T04:24:47,370 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T04:24:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:24:47,447 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T04:24:47,447 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:24:47,449 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:24:47,449 DEBUG [Time-limited test {}] regionserver.HStore(1541): d55d2cfc4b6c9ab5665fa7f430ea318f/info is initiating minor compaction (all files) 2024-11-10T04:24:47,449 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:24:47,449 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:47,449 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of d55d2cfc4b6c9ab5665fa7f430ea318f/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:47,449 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/c1bc8769d89f467d8cecd80d629a0271, hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/21e18c8ff9b249b1ad3c9852f6745f3e, hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4b6e7d7067f745a1bada1976b962dbb6] into tmpdir=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp, totalSize=17.7 K 2024-11-10T04:24:47,450 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c1bc8769d89f467d8cecd80d629a0271, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731212657225 2024-11-10T04:24:47,450 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 21e18c8ff9b249b1ad3c9852f6745f3e, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731212667289 2024-11-10T04:24:47,450 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4b6e7d7067f745a1bada1976b962dbb6, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731212677379 2024-11-10T04:24:47,461 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): d55d2cfc4b6c9ab5665fa7f430ea318f#info#compaction#46 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:24:47,462 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/c149de59c7594577a0a71cf6bdc4e43f is 1080, key is row0001/info:/1731212657225/Put/seqid=0 2024-11-10T04:24:47,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741841_1017 (size=8296) 2024-11-10T04:24:47,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741841_1017 (size=8296) 2024-11-10T04:24:47,472 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/c149de59c7594577a0a71cf6bdc4e43f as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/c149de59c7594577a0a71cf6bdc4e43f 2024-11-10T04:24:47,478 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55d2cfc4b6c9ab5665fa7f430ea318f/info of d55d2cfc4b6c9ab5665fa7f430ea318f into c149de59c7594577a0a71cf6bdc4e43f(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:24:47,478 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for d55d2cfc4b6c9ab5665fa7f430ea318f: 2024-11-10T04:24:47,480 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38659%2C1731212646342.1731212687480 2024-11-10T04:24:47,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:47,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:47,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:47,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:47,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:47,486 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212677380 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212687480 2024-11-10T04:24:47,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38631:38631),(127.0.0.1/127.0.0.1:36307:36307)] 2024-11-10T04:24:47,487 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212677380 is not closed yet, will try archiving it next time 2024-11-10T04:24:47,487 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212646735 to hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/oldWALs/d1d9e3766cb1%2C38659%2C1731212646342.1731212646735 2024-11-10T04:24:47,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741839_1015 (size=2520) 2024-11-10T04:24:47,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:47,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741839_1015 (size=2520) 2024-11-10T04:24:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:24:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T04:24:47,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:24:47,491 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:24:47,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:24:47,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38659 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T04:24:47,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:47,644 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing d55d2cfc4b6c9ab5665fa7f430ea318f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T04:24:47,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/4667915294b64f90a4b5243b9b1c2124 is 1080, key is row0000/info:/1731212687479/Put/seqid=0 2024-11-10T04:24:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741843_1019 (size=6033) 2024-11-10T04:24:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741843_1019 (size=6033) 2024-11-10T04:24:47,654 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/4667915294b64f90a4b5243b9b1c2124 2024-11-10T04:24:47,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/4667915294b64f90a4b5243b9b1c2124 as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4667915294b64f90a4b5243b9b1c2124 2024-11-10T04:24:47,665 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4667915294b64f90a4b5243b9b1c2124, entries=1, sequenceid=18, filesize=5.9 K 2024-11-10T04:24:47,666 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d55d2cfc4b6c9ab5665fa7f430ea318f in 22ms, sequenceid=18, compaction requested=false 2024-11-10T04:24:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for d55d2cfc4b6c9ab5665fa7f430ea318f: 2024-11-10T04:24:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T04:24:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T04:24:47,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T04:24:47,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec 2024-11-10T04:24:47,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec 2024-11-10T04:24:48,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:48,273 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:49,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:49,274 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:50,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:50,274 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:51,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:51,275 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:52,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:52,275 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:52,518 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d55d2cfc4b6c9ab5665fa7f430ea318f, had cached 0 bytes from a total of 14329 2024-11-10T04:24:53,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:53,276 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:54,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:54,276 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:55,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:55,277 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:56,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:56,277 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:57,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:57,278 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T04:24:57,538 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T04:24:57,540 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38659%2C1731212646342.1731212697540 2024-11-10T04:24:57,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,547 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212687480 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212697540 2024-11-10T04:24:57,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741842_1018 (size=2026) 2024-11-10T04:24:57,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741842_1018 (size=2026) 2024-11-10T04:24:57,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38631:38631),(127.0.0.1/127.0.0.1:36307:36307)] 2024-11-10T04:24:57,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212687480 is not closed yet, will try archiving it next time 2024-11-10T04:24:57,549 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/WALs/d1d9e3766cb1,38659,1731212646342/d1d9e3766cb1%2C38659%2C1731212646342.1731212677380 to hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/oldWALs/d1d9e3766cb1%2C38659%2C1731212646342.1731212677380 2024-11-10T04:24:57,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:24:57,550 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:24:57,550 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:24:57,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:57,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:57,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:24:57,550 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:24:57,550 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=804904204, stopped=false 2024-11-10T04:24:57,550 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,43815,1731212646285 2024-11-10T04:24:57,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:24:57,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:24:57,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:57,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:57,552 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:24:57,552 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:24:57,552 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:24:57,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:57,553 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,38659,1731212646342' ***** 2024-11-10T04:24:57,553 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:24:57,553 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:24:57,553 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:24:57,553 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:24:57,553 INFO [RS:0;d1d9e3766cb1:38659 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:24:57,553 INFO [RS:0;d1d9e3766cb1:38659 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:24:57,553 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:57,553 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:38659. 2024-11-10T04:24:57,554 DEBUG [RS:0;d1d9e3766cb1:38659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d55d2cfc4b6c9ab5665fa7f430ea318f, disabling compactions & flushes 2024-11-10T04:24:57,554 DEBUG [RS:0;d1d9e3766cb1:38659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:57,554 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. after waiting 0 ms 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:24:57,554 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d55d2cfc4b6c9ab5665fa7f430ea318f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T04:24:57,554 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T04:24:57,554 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1325): Online Regions={d55d2cfc4b6c9ab5665fa7f430ea318f=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T04:24:57,554 DEBUG [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d55d2cfc4b6c9ab5665fa7f430ea318f 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:24:57,554 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:24:57,554 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:24:57,555 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-10T04:24:57,558 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/f8bf7f8b8b0a4932b52c351bd7a29fad is 1080, key is row0001/info:/1731212697539/Put/seqid=0 2024-11-10T04:24:57,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741845_1021 (size=6033) 2024-11-10T04:24:57,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741845_1021 (size=6033) 2024-11-10T04:24:57,564 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/f8bf7f8b8b0a4932b52c351bd7a29fad 2024-11-10T04:24:57,570 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/.tmp/info/f8bf7f8b8b0a4932b52c351bd7a29fad as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/f8bf7f8b8b0a4932b52c351bd7a29fad 2024-11-10T04:24:57,571 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/info/22ae1cb3f4f449d094ebd0f762c94daa is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f./info:regioninfo/1731212647531/Put/seqid=0 2024-11-10T04:24:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741846_1022 (size=7308) 2024-11-10T04:24:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741846_1022 (size=7308) 2024-11-10T04:24:57,576 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/info/22ae1cb3f4f449d094ebd0f762c94daa 2024-11-10T04:24:57,577 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/f8bf7f8b8b0a4932b52c351bd7a29fad, entries=1, sequenceid=22, filesize=5.9 K 2024-11-10T04:24:57,578 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d55d2cfc4b6c9ab5665fa7f430ea318f in 24ms, sequenceid=22, compaction requested=true 2024-11-10T04:24:57,579 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/c1bc8769d89f467d8cecd80d629a0271, hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/21e18c8ff9b249b1ad3c9852f6745f3e, hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4b6e7d7067f745a1bada1976b962dbb6] to archive 2024-11-10T04:24:57,580 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:24:57,582 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/c1bc8769d89f467d8cecd80d629a0271 to hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/c1bc8769d89f467d8cecd80d629a0271 2024-11-10T04:24:57,583 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/21e18c8ff9b249b1ad3c9852f6745f3e to hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/21e18c8ff9b249b1ad3c9852f6745f3e 2024-11-10T04:24:57,584 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4b6e7d7067f745a1bada1976b962dbb6 to hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/info/4b6e7d7067f745a1bada1976b962dbb6 2024-11-10T04:24:57,585 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d1d9e3766cb1:43815 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T04:24:57,585 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c1bc8769d89f467d8cecd80d629a0271=6033, 21e18c8ff9b249b1ad3c9852f6745f3e=6033, 4b6e7d7067f745a1bada1976b962dbb6=6033] 2024-11-10T04:24:57,592 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d55d2cfc4b6c9ab5665fa7f430ea318f/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-10T04:24:57,593 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:57,593 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d55d2cfc4b6c9ab5665fa7f430ea318f: Waiting for close lock at 1731212697554Running coprocessor pre-close hooks at 1731212697554Disabling compacts and flushes for region at 1731212697554Disabling writes for close at 1731212697554Obtaining lock to block concurrent updates at 1731212697554Preparing flush snapshotting stores in d55d2cfc4b6c9ab5665fa7f430ea318f at 1731212697554Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731212697554Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. at 1731212697555 (+1 ms)Flushing d55d2cfc4b6c9ab5665fa7f430ea318f/info: creating writer at 1731212697555Flushing d55d2cfc4b6c9ab5665fa7f430ea318f/info: appending metadata at 1731212697558 (+3 ms)Flushing d55d2cfc4b6c9ab5665fa7f430ea318f/info: closing flushed file at 1731212697558Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e190f12: reopening flushed file at 1731212697570 (+12 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d55d2cfc4b6c9ab5665fa7f430ea318f in 24ms, sequenceid=22, compaction requested=true at 1731212697578 (+8 ms)Writing region close event to WAL at 1731212697585 (+7 ms)Running coprocessor post-close hooks at 1731212697593 (+8 ms)Closed at 1731212697593 2024-11-10T04:24:57,593 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731212647177.d55d2cfc4b6c9ab5665fa7f430ea318f. 2024-11-10T04:24:57,595 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/ns/a05311a9098944f29f7ff64a1b93a8e0 is 43, key is default/ns:d/1731212647148/Put/seqid=0 2024-11-10T04:24:57,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741847_1023 (size=5153) 2024-11-10T04:24:57,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741847_1023 (size=5153) 2024-11-10T04:24:57,600 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/ns/a05311a9098944f29f7ff64a1b93a8e0 2024-11-10T04:24:57,619 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/table/dd12707ba1874137ad83e79e61f9d767 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731212647541/Put/seqid=0 2024-11-10T04:24:57,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741848_1024 (size=5508) 2024-11-10T04:24:57,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741848_1024 (size=5508) 2024-11-10T04:24:57,624 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/table/dd12707ba1874137ad83e79e61f9d767 2024-11-10T04:24:57,629 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/info/22ae1cb3f4f449d094ebd0f762c94daa as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/info/22ae1cb3f4f449d094ebd0f762c94daa 2024-11-10T04:24:57,634 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/info/22ae1cb3f4f449d094ebd0f762c94daa, entries=10, sequenceid=11, filesize=7.1 K 2024-11-10T04:24:57,635 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/ns/a05311a9098944f29f7ff64a1b93a8e0 as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/ns/a05311a9098944f29f7ff64a1b93a8e0 2024-11-10T04:24:57,640 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/ns/a05311a9098944f29f7ff64a1b93a8e0, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T04:24:57,641 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/.tmp/table/dd12707ba1874137ad83e79e61f9d767 as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/table/dd12707ba1874137ad83e79e61f9d767 2024-11-10T04:24:57,646 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/table/dd12707ba1874137ad83e79e61f9d767, entries=2, sequenceid=11, filesize=5.4 K 2024-11-10T04:24:57,647 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false 2024-11-10T04:24:57,649 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T04:24:57,649 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T04:24:57,652 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T04:24:57,652 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:24:57,653 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:24:57,653 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212697554Running coprocessor pre-close hooks at 1731212697554Disabling compacts and flushes for region at 1731212697554Disabling writes for close at 1731212697554Obtaining lock to block concurrent updates at 1731212697555 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731212697555Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731212697555Flushing stores of hbase:meta,,1.1588230740 at 1731212697555Flushing 1588230740/info: creating writer at 1731212697556 (+1 ms)Flushing 1588230740/info: appending metadata at 1731212697570 (+14 ms)Flushing 1588230740/info: closing flushed file at 1731212697570Flushing 1588230740/ns: creating writer at 1731212697581 (+11 ms)Flushing 1588230740/ns: appending metadata at 1731212697595 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731212697595Flushing 1588230740/table: creating writer at 1731212697605 (+10 ms)Flushing 1588230740/table: appending metadata at 1731212697619 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731212697619Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f78341f: reopening flushed file at 1731212697629 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@694c340: reopening flushed file at 1731212697634 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17089329: reopening flushed file at 1731212697640 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false at 1731212697647 (+7 ms)Writing region close event to WAL at 1731212697648 (+1 ms)Running coprocessor post-close hooks at 1731212697652 (+4 ms)Closed at 1731212697653 (+1 ms) 2024-11-10T04:24:57,653 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:24:57,754 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,38659,1731212646342; all regions closed. 2024-11-10T04:24:57,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741834_1010 (size=3306) 2024-11-10T04:24:57,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741834_1010 (size=3306) 2024-11-10T04:24:57,760 DEBUG [RS:0;d1d9e3766cb1:38659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/oldWALs 2024-11-10T04:24:57,760 INFO [RS:0;d1d9e3766cb1:38659 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38659%2C1731212646342.meta:.meta(num 1731212647108) 2024-11-10T04:24:57,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741844_1020 (size=1252) 2024-11-10T04:24:57,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741844_1020 (size=1252) 2024-11-10T04:24:57,765 DEBUG [RS:0;d1d9e3766cb1:38659 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/oldWALs 2024-11-10T04:24:57,765 INFO [RS:0;d1d9e3766cb1:38659 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38659%2C1731212646342:(num 1731212697540) 2024-11-10T04:24:57,765 DEBUG [RS:0;d1d9e3766cb1:38659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:57,765 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:24:57,765 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:24:57,765 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:24:57,766 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:24:57,766 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:24:57,766 INFO [RS:0;d1d9e3766cb1:38659 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38659 2024-11-10T04:24:57,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,38659,1731212646342 2024-11-10T04:24:57,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:24:57,768 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:24:57,769 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,38659,1731212646342] 2024-11-10T04:24:57,771 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,38659,1731212646342 already deleted, retry=false 2024-11-10T04:24:57,771 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,38659,1731212646342 expired; onlineServers=0 2024-11-10T04:24:57,771 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,43815,1731212646285' ***** 2024-11-10T04:24:57,771 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:24:57,771 INFO [M:0;d1d9e3766cb1:43815 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:24:57,771 INFO [M:0;d1d9e3766cb1:43815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:24:57,771 DEBUG [M:0;d1d9e3766cb1:43815 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:24:57,771 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:24:57,771 DEBUG [M:0;d1d9e3766cb1:43815 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:24:57,771 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212646505 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212646505,5,FailOnTimeoutGroup] 2024-11-10T04:24:57,771 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212646505 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212646505,5,FailOnTimeoutGroup] 2024-11-10T04:24:57,771 INFO [M:0;d1d9e3766cb1:43815 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:24:57,771 INFO [M:0;d1d9e3766cb1:43815 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:24:57,771 DEBUG [M:0;d1d9e3766cb1:43815 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:24:57,771 INFO [M:0;d1d9e3766cb1:43815 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:24:57,771 INFO [M:0;d1d9e3766cb1:43815 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:24:57,772 INFO [M:0;d1d9e3766cb1:43815 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:24:57,772 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:24:57,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:24:57,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:57,773 DEBUG [M:0;d1d9e3766cb1:43815 {}] zookeeper.ZKUtil(347): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:24:57,773 WARN [M:0;d1d9e3766cb1:43815 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:24:57,773 INFO [M:0;d1d9e3766cb1:43815 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/.lastflushedseqids 2024-11-10T04:24:57,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741849_1025 (size=130) 2024-11-10T04:24:57,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741849_1025 (size=130) 2024-11-10T04:24:57,779 INFO [M:0;d1d9e3766cb1:43815 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:24:57,779 INFO [M:0;d1d9e3766cb1:43815 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:24:57,779 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:24:57,779 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:57,779 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:57,779 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:24:57,779 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:57,779 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB 2024-11-10T04:24:57,795 DEBUG [M:0;d1d9e3766cb1:43815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/667a72855ac44c37975885a70d3b126e is 82, key is hbase:meta,,1/info:regioninfo/1731212647131/Put/seqid=0 2024-11-10T04:24:57,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741850_1026 (size=5672) 2024-11-10T04:24:57,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741850_1026 (size=5672) 2024-11-10T04:24:57,799 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/667a72855ac44c37975885a70d3b126e 2024-11-10T04:24:57,818 DEBUG [M:0;d1d9e3766cb1:43815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9c3ce2a33b7944559c0cd16356ebcf1e is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731212647545/Put/seqid=0 2024-11-10T04:24:57,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741851_1027 (size=7824) 2024-11-10T04:24:57,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741851_1027 (size=7824) 2024-11-10T04:24:57,823 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9c3ce2a33b7944559c0cd16356ebcf1e 2024-11-10T04:24:57,827 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9c3ce2a33b7944559c0cd16356ebcf1e 2024-11-10T04:24:57,841 DEBUG [M:0;d1d9e3766cb1:43815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5bc6a819a32b4472a1862b89750d24ba is 69, key is d1d9e3766cb1,38659,1731212646342/rs:state/1731212646586/Put/seqid=0 2024-11-10T04:24:57,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741852_1028 (size=5156) 2024-11-10T04:24:57,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741852_1028 (size=5156) 2024-11-10T04:24:57,847 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5bc6a819a32b4472a1862b89750d24ba 2024-11-10T04:24:57,865 DEBUG [M:0;d1d9e3766cb1:43815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/11b962fcf6dd4d99b07a5c47b1e9056a is 52, key is load_balancer_on/state:d/1731212647173/Put/seqid=0 2024-11-10T04:24:57,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741853_1029 (size=5056) 2024-11-10T04:24:57,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741853_1029 (size=5056) 2024-11-10T04:24:57,869 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/11b962fcf6dd4d99b07a5c47b1e9056a 2024-11-10T04:24:57,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:57,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x101906c0ce60001, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:57,870 INFO [RS:0;d1d9e3766cb1:38659 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:24:57,870 INFO [RS:0;d1d9e3766cb1:38659 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,38659,1731212646342; zookeeper connection closed. 2024-11-10T04:24:57,870 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b2a83e4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b2a83e4 2024-11-10T04:24:57,870 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:24:57,874 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/667a72855ac44c37975885a70d3b126e as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/667a72855ac44c37975885a70d3b126e 2024-11-10T04:24:57,878 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/667a72855ac44c37975885a70d3b126e, entries=8, sequenceid=121, filesize=5.5 K 2024-11-10T04:24:57,879 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9c3ce2a33b7944559c0cd16356ebcf1e as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9c3ce2a33b7944559c0cd16356ebcf1e 2024-11-10T04:24:57,884 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9c3ce2a33b7944559c0cd16356ebcf1e 2024-11-10T04:24:57,884 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9c3ce2a33b7944559c0cd16356ebcf1e, entries=14, sequenceid=121, filesize=7.6 K 2024-11-10T04:24:57,885 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5bc6a819a32b4472a1862b89750d24ba as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5bc6a819a32b4472a1862b89750d24ba 2024-11-10T04:24:57,889 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5bc6a819a32b4472a1862b89750d24ba, entries=1, sequenceid=121, filesize=5.0 K 2024-11-10T04:24:57,890 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/11b962fcf6dd4d99b07a5c47b1e9056a as hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/11b962fcf6dd4d99b07a5c47b1e9056a 2024-11-10T04:24:57,895 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33403/user/jenkins/test-data/f04d1c9b-ee54-df33-752f-312854e36772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/11b962fcf6dd4d99b07a5c47b1e9056a, entries=1, sequenceid=121, filesize=4.9 K 2024-11-10T04:24:57,896 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false 2024-11-10T04:24:57,898 INFO [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:57,898 DEBUG [M:0;d1d9e3766cb1:43815 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212697779Disabling compacts and flushes for region at 1731212697779Disabling writes for close at 1731212697779Obtaining lock to block concurrent updates at 1731212697779Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212697779Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44647, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1731212697780 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212697780Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212697780Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212697794 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212697794Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212697804 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212697818 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212697818Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212697827 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212697841 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212697841Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731212697851 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731212697864 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731212697864Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a13e9f8: reopening flushed file at 1731212697873 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36be4717: reopening flushed file at 1731212697879 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73bd3881: reopening flushed file at 1731212697884 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bd7631f: reopening flushed file at 1731212697889 (+5 ms)Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false at 1731212697896 (+7 ms)Writing region close event to WAL at 1731212697898 (+2 ms)Closed at 1731212697898 2024-11-10T04:24:57,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:24:57,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741830_1006 (size=53044) 2024-11-10T04:24:57,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39271 is added to blk_1073741830_1006 (size=53044) 2024-11-10T04:24:57,902 INFO [M:0;d1d9e3766cb1:43815 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:24:57,902 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:24:57,902 INFO [M:0;d1d9e3766cb1:43815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43815 2024-11-10T04:24:57,902 INFO [M:0;d1d9e3766cb1:43815 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:24:58,004 INFO [M:0;d1d9e3766cb1:43815 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:24:58,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:58,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43815-0x101906c0ce60000, quorum=127.0.0.1:56992, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:24:58,006 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6007e850{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:58,007 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@18701e65{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:24:58,007 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:24:58,007 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8afa355{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:24:58,007 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35023063{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:24:58,009 WARN [BP-295375615-172.17.0.2-1731212645642 heartbeating to localhost/127.0.0.1:33403 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:24:58,009 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:24:58,009 WARN [BP-295375615-172.17.0.2-1731212645642 heartbeating to localhost/127.0.0.1:33403 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-295375615-172.17.0.2-1731212645642 (Datanode Uuid b358d3e2-e6ce-412b-b611-85fbdffc1833) service to localhost/127.0.0.1:33403 2024-11-10T04:24:58,009 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:24:58,010 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data3/current/BP-295375615-172.17.0.2-1731212645642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:58,010 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data4/current/BP-295375615-172.17.0.2-1731212645642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:58,010 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:24:58,012 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67c2b9b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:58,012 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6dc3ea71{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:24:58,012 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:24:58,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f08894b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:24:58,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3df88721{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:24:58,014 WARN [BP-295375615-172.17.0.2-1731212645642 heartbeating to localhost/127.0.0.1:33403 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:24:58,014 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:24:58,014 WARN [BP-295375615-172.17.0.2-1731212645642 heartbeating to localhost/127.0.0.1:33403 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-295375615-172.17.0.2-1731212645642 (Datanode Uuid f27a0ae3-ef4d-4e66-af48-4ac02730ba0c) service to localhost/127.0.0.1:33403 2024-11-10T04:24:58,014 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:24:58,015 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data1/current/BP-295375615-172.17.0.2-1731212645642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:58,015 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/cluster_6530222f-516d-9359-3953-3e93c53f2887/data/data2/current/BP-295375615-172.17.0.2-1731212645642 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:24:58,015 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:24:58,021 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c447ff3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:24:58,022 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@301adf20{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:24:58,022 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:24:58,022 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dd613b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:24:58,022 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3cb009a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir/,STOPPED} 2024-11-10T04:24:58,029 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:24:58,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:24:58,053 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 179) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33403 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33403 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:33403 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33403 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33403 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33403 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33403 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/d1d9e3766cb1:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:33403 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33403 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=485 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=55 (was 116), ProcessCount=11 (was 11), AvailableMemoryMB=6393 (was 6429) 2024-11-10T04:24:58,060 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=55, ProcessCount=11, AvailableMemoryMB=6393 2024-11-10T04:24:58,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:24:58,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.log.dir so I do NOT create it in target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e 2024-11-10T04:24:58,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/51f8c72d-e13d-c9de-06a4-4f966cca99cc/hadoop.tmp.dir so I do NOT create it in target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e 2024-11-10T04:24:58,060 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0, deleteOnExit=true 2024-11-10T04:24:58,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/test.cache.data in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:24:58,061 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:24:58,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:24:58,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:24:58,075 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:24:58,129 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:58,132 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:24:58,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:24:58,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:24:58,133 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:24:58,134 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:58,134 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35e03861{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:24:58,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a0fc9ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:24:58,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:58,249 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@319ddf98{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/java.io.tmpdir/jetty-localhost-45925-hadoop-hdfs-3_4_1-tests_jar-_-any-12089048769338696361/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:24:58,250 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@371dfe16{HTTP/1.1, (http/1.1)}{localhost:45925} 2024-11-10T04:24:58,250 INFO [Time-limited test {}] server.Server(415): Started @237795ms 2024-11-10T04:24:58,263 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:24:58,278 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:58,331 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:58,334 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:24:58,335 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:24:58,335 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:24:58,335 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:24:58,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@374d3611{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:24:58,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ecf816b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:24:58,448 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@444d0b71{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/java.io.tmpdir/jetty-localhost-34265-hadoop-hdfs-3_4_1-tests_jar-_-any-1663618365907110642/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:58,448 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3bc081d8{HTTP/1.1, (http/1.1)}{localhost:34265} 2024-11-10T04:24:58,448 INFO [Time-limited test {}] server.Server(415): Started @237993ms 2024-11-10T04:24:58,449 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:24:58,478 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:24:58,481 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:24:58,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:24:58,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:24:58,482 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:24:58,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64d2170c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:24:58,483 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41a74ab6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:24:58,536 WARN [Thread-1970 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data2/current/BP-1653749568-172.17.0.2-1731212698081/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:58,536 WARN [Thread-1969 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data1/current/BP-1653749568-172.17.0.2-1731212698081/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:58,552 WARN [Thread-1948 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:24:58,555 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb7cb97b9ac8c8542 with lease ID 0xc2d9334c092e3ddd: Processing first storage report for DS-2501d6d7-2b0e-4528-b627-6073902aa354 from datanode DatanodeRegistration(127.0.0.1:44267, datanodeUuid=e0caa0eb-56b4-4a9a-b9a6-08ef6c379ac1, infoPort=37577, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081) 2024-11-10T04:24:58,555 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb7cb97b9ac8c8542 with lease ID 0xc2d9334c092e3ddd: from storage DS-2501d6d7-2b0e-4528-b627-6073902aa354 node DatanodeRegistration(127.0.0.1:44267, datanodeUuid=e0caa0eb-56b4-4a9a-b9a6-08ef6c379ac1, infoPort=37577, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:58,555 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb7cb97b9ac8c8542 with lease ID 0xc2d9334c092e3ddd: Processing first storage report for DS-5d6a44f2-7e93-4bc8-8f5e-ad7246113d58 from datanode DatanodeRegistration(127.0.0.1:44267, datanodeUuid=e0caa0eb-56b4-4a9a-b9a6-08ef6c379ac1, infoPort=37577, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081) 2024-11-10T04:24:58,555 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb7cb97b9ac8c8542 with lease ID 0xc2d9334c092e3ddd: from storage DS-5d6a44f2-7e93-4bc8-8f5e-ad7246113d58 node DatanodeRegistration(127.0.0.1:44267, datanodeUuid=e0caa0eb-56b4-4a9a-b9a6-08ef6c379ac1, infoPort=37577, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:58,602 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:24:58,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18d1ee92{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/java.io.tmpdir/jetty-localhost-35141-hadoop-hdfs-3_4_1-tests_jar-_-any-13298576131505909920/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:24:58,605 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78512cf7{HTTP/1.1, (http/1.1)}{localhost:35141} 2024-11-10T04:24:58,605 INFO [Time-limited test {}] server.Server(415): Started @238151ms 2024-11-10T04:24:58,607 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:24:58,694 WARN [Thread-1995 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data3/current/BP-1653749568-172.17.0.2-1731212698081/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:58,694 WARN [Thread-1996 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data4/current/BP-1653749568-172.17.0.2-1731212698081/current, will proceed with Du for space computation calculation, 2024-11-10T04:24:58,716 WARN [Thread-1984 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:24:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa85915a15b0e5b4 with lease ID 0xc2d9334c092e3dde: Processing first storage report for DS-f4916fa3-71ae-4070-9297-6486fb43473f from datanode DatanodeRegistration(127.0.0.1:40347, datanodeUuid=8b4f7f24-ba32-4fcc-86cd-1a57d75c6712, infoPort=44489, infoSecurePort=0, ipcPort=43771, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081) 2024-11-10T04:24:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa85915a15b0e5b4 with lease ID 0xc2d9334c092e3dde: from storage DS-f4916fa3-71ae-4070-9297-6486fb43473f node DatanodeRegistration(127.0.0.1:40347, datanodeUuid=8b4f7f24-ba32-4fcc-86cd-1a57d75c6712, infoPort=44489, infoSecurePort=0, ipcPort=43771, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa85915a15b0e5b4 with lease ID 0xc2d9334c092e3dde: Processing first storage report for DS-096bdc97-8a6e-479c-b20d-37d09738636d from datanode DatanodeRegistration(127.0.0.1:40347, datanodeUuid=8b4f7f24-ba32-4fcc-86cd-1a57d75c6712, infoPort=44489, infoSecurePort=0, ipcPort=43771, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081) 2024-11-10T04:24:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa85915a15b0e5b4 with lease ID 0xc2d9334c092e3dde: from storage DS-096bdc97-8a6e-479c-b20d-37d09738636d node DatanodeRegistration(127.0.0.1:40347, datanodeUuid=8b4f7f24-ba32-4fcc-86cd-1a57d75c6712, infoPort=44489, infoSecurePort=0, ipcPort=43771, storageInfo=lv=-57;cid=testClusterID;nsid=944556698;c=1731212698081), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:24:58,728 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e 2024-11-10T04:24:58,730 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/zookeeper_0, clientPort=49477, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:24:58,731 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49477 2024-11-10T04:24:58,731 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:58,733 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:58,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:24:58,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:24:58,742 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2 with version=8 2024-11-10T04:24:58,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase-staging 2024-11-10T04:24:58,744 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:24:58,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:58,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:58,744 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:24:58,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:58,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:24:58,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:24:58,744 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:24:58,745 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40483 2024-11-10T04:24:58,746 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40483 connecting to ZooKeeper ensemble=127.0.0.1:49477 2024-11-10T04:24:58,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404830x0, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:24:58,753 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40483-0x101906cd9d20000 connected 2024-11-10T04:24:58,768 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:58,769 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:58,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:24:58,771 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2, hbase.cluster.distributed=false 2024-11-10T04:24:58,773 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:24:58,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40483 2024-11-10T04:24:58,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40483 2024-11-10T04:24:58,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40483 2024-11-10T04:24:58,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40483 2024-11-10T04:24:58,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40483 2024-11-10T04:24:58,789 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:24:58,789 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:58,789 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:58,789 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:24:58,789 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:24:58,789 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:24:58,789 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:24:58,789 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:24:58,790 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38277 2024-11-10T04:24:58,791 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38277 connecting to ZooKeeper ensemble=127.0.0.1:49477 2024-11-10T04:24:58,791 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:58,793 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:58,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382770x0, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:24:58,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38277-0x101906cd9d20001 connected 2024-11-10T04:24:58,797 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:24:58,797 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:24:58,798 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:24:58,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:24:58,799 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:24:58,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38277 2024-11-10T04:24:58,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38277 2024-11-10T04:24:58,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38277 2024-11-10T04:24:58,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38277 2024-11-10T04:24:58,801 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38277 2024-11-10T04:24:58,812 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:40483 2024-11-10T04:24:58,812 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:58,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:58,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:58,814 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:58,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:24:58,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,816 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:24:58,817 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,40483,1731212698744 from backup master directory 2024-11-10T04:24:58,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:58,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:58,818 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:24:58,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:24:58,818 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:58,822 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/hbase.id] with ID: 72e92be7-50ab-4744-af36-017680c49188 2024-11-10T04:24:58,822 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/.tmp/hbase.id 2024-11-10T04:24:58,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:24:58,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:24:58,828 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/.tmp/hbase.id]:[hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/hbase.id] 2024-11-10T04:24:58,838 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:58,838 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:24:58,840 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-10T04:24:58,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:24:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:24:58,848 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:24:58,849 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:24:58,849 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:24:58,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:24:58,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:24:58,857 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store 2024-11-10T04:24:58,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:24:58,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:24:58,863 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:58,863 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:24:58,863 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:58,863 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:58,863 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:24:58,863 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:58,863 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:24:58,863 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212698863Disabling compacts and flushes for region at 1731212698863Disabling writes for close at 1731212698863Writing region close event to WAL at 1731212698863Closed at 1731212698863 2024-11-10T04:24:58,864 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/.initializing 2024-11-10T04:24:58,864 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/WALs/d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:58,866 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C40483%2C1731212698744, suffix=, logDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/WALs/d1d9e3766cb1,40483,1731212698744, archiveDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/oldWALs, maxLogs=10 2024-11-10T04:24:58,867 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C40483%2C1731212698744.1731212698866 2024-11-10T04:24:58,872 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/WALs/d1d9e3766cb1,40483,1731212698744/d1d9e3766cb1%2C40483%2C1731212698744.1731212698866 2024-11-10T04:24:58,876 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44489:44489),(127.0.0.1/127.0.0.1:37577:37577)] 2024-11-10T04:24:58,878 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:24:58,878 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:58,878 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,879 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:24:58,881 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:58,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:24:58,883 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:58,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:24:58,884 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:58,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:24:58,886 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:58,886 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,887 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,887 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,888 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,888 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,889 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:24:58,890 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:24:58,892 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:24:58,892 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=690218, jitterRate=-0.1223425418138504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:24:58,893 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212698879Initializing all the Stores at 1731212698879Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212698879Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212698880 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212698880Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212698880Cleaning up temporary data from old regions at 1731212698888 (+8 ms)Region opened successfully at 1731212698893 (+5 ms) 2024-11-10T04:24:58,893 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:24:58,896 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f1ef29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:24:58,897 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:24:58,897 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:24:58,897 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:24:58,897 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:24:58,897 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T04:24:58,898 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T04:24:58,898 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:24:58,899 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:24:58,900 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:24:58,901 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:24:58,902 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:24:58,902 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:24:58,904 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:24:58,904 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:24:58,905 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:24:58,906 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:24:58,907 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:24:58,908 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:24:58,910 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:24:58,912 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:24:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:24:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:24:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,916 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,40483,1731212698744, sessionid=0x101906cd9d20000, setting cluster-up flag (Was=false) 2024-11-10T04:24:58,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,925 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:24:58,926 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:58,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:58,934 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:24:58,935 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:58,936 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:24:58,937 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:58,938 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:24:58,938 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:24:58,938 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,40483,1731212698744 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:24:58,939 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212728941 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:24:58,941 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:58,941 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:24:58,941 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:24:58,942 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,942 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212698942,5,FailOnTimeoutGroup] 2024-11-10T04:24:58,942 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212698942,5,FailOnTimeoutGroup] 2024-11-10T04:24:58,942 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:58,942 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:58,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:24:58,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:24:58,951 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:24:58,952 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2 2024-11-10T04:24:58,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:24:58,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:24:58,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:58,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:24:58,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:24:58,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:58,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:24:58,964 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:24:58,965 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:58,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:24:58,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:24:58,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:58,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:24:58,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:24:58,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:58,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:58,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:24:58,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740 2024-11-10T04:24:58,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740 2024-11-10T04:24:58,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:24:58,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:24:58,970 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:24:58,971 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:24:58,973 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:24:58,974 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741462, jitterRate=-0.05718272924423218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:24:58,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212698960Initializing all the Stores at 1731212698961 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212698961Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212698961Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212698961Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212698961Cleaning up temporary data from old regions at 1731212698970 (+9 ms)Region opened successfully at 1731212698974 (+4 ms) 2024-11-10T04:24:58,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:24:58,974 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:24:58,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:24:58,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:24:58,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:24:58,975 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:24:58,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212698974Disabling compacts and flushes for region at 1731212698974Disabling writes for close at 1731212698974Writing region close event to WAL at 1731212698975 (+1 ms)Closed at 1731212698975 2024-11-10T04:24:58,976 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:58,976 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:24:58,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:24:58,978 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:24:58,979 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:24:59,002 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(746): ClusterId : 72e92be7-50ab-4744-af36-017680c49188 2024-11-10T04:24:59,002 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:24:59,005 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:24:59,005 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:24:59,007 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:24:59,008 DEBUG [RS:0;d1d9e3766cb1:38277 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c76f416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:24:59,019 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:38277 2024-11-10T04:24:59,019 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:24:59,019 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:24:59,019 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:24:59,020 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,40483,1731212698744 with port=38277, startcode=1731212698789 2024-11-10T04:24:59,020 DEBUG [RS:0;d1d9e3766cb1:38277 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:24:59,022 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42191, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:24:59,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,024 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2 2024-11-10T04:24:59,024 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34945 2024-11-10T04:24:59,024 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:24:59,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:24:59,027 DEBUG [RS:0;d1d9e3766cb1:38277 {}] zookeeper.ZKUtil(111): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,027 WARN [RS:0;d1d9e3766cb1:38277 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:24:59,027 INFO [RS:0;d1d9e3766cb1:38277 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:24:59,027 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,027 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,38277,1731212698789] 2024-11-10T04:24:59,030 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:24:59,032 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:24:59,032 INFO [RS:0;d1d9e3766cb1:38277 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:24:59,032 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,032 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:24:59,033 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:24:59,033 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,033 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,034 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,034 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:24:59,034 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:24:59,034 DEBUG [RS:0;d1d9e3766cb1:38277 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:24:59,034 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,034 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,034 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,034 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,034 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,034 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38277,1731212698789-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:24:59,049 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:24:59,049 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,38277,1731212698789-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,049 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,049 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.Replication(171): d1d9e3766cb1,38277,1731212698789 started 2024-11-10T04:24:59,062 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,062 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,38277,1731212698789, RpcServer on d1d9e3766cb1/172.17.0.2:38277, sessionid=0x101906cd9d20001 2024-11-10T04:24:59,062 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:24:59,062 DEBUG [RS:0;d1d9e3766cb1:38277 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,062 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38277,1731212698789' 2024-11-10T04:24:59,062 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:24:59,063 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:24:59,063 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:24:59,063 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:24:59,063 DEBUG [RS:0;d1d9e3766cb1:38277 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,063 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,38277,1731212698789' 2024-11-10T04:24:59,063 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:24:59,063 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:24:59,064 DEBUG [RS:0;d1d9e3766cb1:38277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:24:59,064 INFO [RS:0;d1d9e3766cb1:38277 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:24:59,064 INFO [RS:0;d1d9e3766cb1:38277 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:24:59,129 WARN [d1d9e3766cb1:40483 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:24:59,166 INFO [RS:0;d1d9e3766cb1:38277 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38277%2C1731212698789, suffix=, logDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789, archiveDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/oldWALs, maxLogs=32 2024-11-10T04:24:59,166 INFO [RS:0;d1d9e3766cb1:38277 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38277%2C1731212698789.1731212699166 2024-11-10T04:24:59,171 INFO [RS:0;d1d9e3766cb1:38277 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212699166 2024-11-10T04:24:59,172 DEBUG [RS:0;d1d9e3766cb1:38277 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37577:37577),(127.0.0.1/127.0.0.1:44489:44489)] 2024-11-10T04:24:59,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:59,279 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:24:59,379 DEBUG [d1d9e3766cb1:40483 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:24:59,380 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,381 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38277,1731212698789, state=OPENING 2024-11-10T04:24:59,383 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:24:59,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:59,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:24:59,385 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:24:59,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:59,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:59,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38277,1731212698789}] 2024-11-10T04:24:59,538 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:24:59,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:24:59,543 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:24:59,543 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:24:59,545 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C38277%2C1731212698789.meta, suffix=.meta, logDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789, archiveDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/oldWALs, maxLogs=32 2024-11-10T04:24:59,545 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38277%2C1731212698789.meta.1731212699545.meta 2024-11-10T04:24:59,553 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.meta.1731212699545.meta 2024-11-10T04:24:59,556 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44489:44489),(127.0.0.1/127.0.0.1:37577:37577)] 2024-11-10T04:24:59,558 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:24:59,559 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:24:59,559 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:24:59,559 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:24:59,559 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:24:59,559 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:59,559 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:24:59,559 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:24:59,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:24:59,561 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:24:59,561 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:59,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:59,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:24:59,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:24:59,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:59,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:59,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:24:59,563 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:24:59,563 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:59,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:59,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:24:59,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:24:59,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:59,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:24:59,565 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:24:59,565 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740 2024-11-10T04:24:59,566 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740 2024-11-10T04:24:59,567 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:24:59,567 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:24:59,567 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:24:59,569 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:24:59,569 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881398, jitterRate=0.12075628340244293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:24:59,569 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:24:59,570 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212699559Writing region info on filesystem at 1731212699559Initializing all the Stores at 1731212699560 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212699560Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212699560Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212699560Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212699560Cleaning up temporary data from old regions at 1731212699567 (+7 ms)Running coprocessor post-open hooks at 1731212699569 (+2 ms)Region opened successfully at 1731212699570 (+1 ms) 2024-11-10T04:24:59,571 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212699538 2024-11-10T04:24:59,573 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:24:59,573 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:24:59,574 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,575 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,38277,1731212698789, state=OPEN 2024-11-10T04:24:59,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:24:59,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:24:59,580 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:59,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:24:59,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:24:59,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,38277,1731212698789 in 195 msec 2024-11-10T04:24:59,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:24:59,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec 2024-11-10T04:24:59,586 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:24:59,586 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:24:59,587 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:24:59,587 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38277,1731212698789, seqNum=-1] 2024-11-10T04:24:59,587 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:24:59,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:24:59,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 656 msec 2024-11-10T04:24:59,594 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212699594, completionTime=-1 2024-11-10T04:24:59,594 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:24:59,594 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:24:59,596 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:24:59,596 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212759596 2024-11-10T04:24:59,596 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731212819596 2024-11-10T04:24:59,596 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T04:24:59,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40483,1731212698744-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40483,1731212698744-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40483,1731212698744-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:40483, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,598 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:24:59,600 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.782sec 2024-11-10T04:24:59,601 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:24:59,601 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:24:59,601 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:24:59,601 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:24:59,601 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:24:59,601 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40483,1731212698744-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:24:59,601 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40483,1731212698744-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:24:59,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea28eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:24:59,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,40483,-1 for getting cluster id 2024-11-10T04:24:59,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:24:59,603 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:24:59,603 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:24:59,603 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,40483,1731212698744-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:24:59,604 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '72e92be7-50ab-4744-af36-017680c49188' 2024-11-10T04:24:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:24:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "72e92be7-50ab-4744-af36-017680c49188" 2024-11-10T04:24:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6eae0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:24:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,40483,-1] 2024-11-10T04:24:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:24:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:24:59,606 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:24:59,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d059d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:24:59,607 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:24:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,38277,1731212698789, seqNum=-1] 2024-11-10T04:24:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:24:59,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:24:59,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:59,611 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:24:59,613 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T04:24:59,613 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:24:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,40483,1731212698744 2024-11-10T04:24:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54cc079f 2024-11-10T04:24:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:24:59,615 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:24:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40483 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T04:24:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T04:24:59,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:24:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40483 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-10T04:24:59,618 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:24:59,619 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:59,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-10T04:24:59,620 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:24:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:24:59,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741835_1011 (size=381) 2024-11-10T04:24:59,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741835_1011 (size=381) 2024-11-10T04:24:59,631 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c285b8b061af9a7de1e4f817de4f031, NAME => 'TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2 2024-11-10T04:24:59,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741836_1012 (size=64) 2024-11-10T04:24:59,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741836_1012 (size=64) 2024-11-10T04:24:59,640 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:59,640 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 4c285b8b061af9a7de1e4f817de4f031, disabling compactions & flushes 2024-11-10T04:24:59,640 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:24:59,640 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:24:59,640 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. after waiting 0 ms 2024-11-10T04:24:59,640 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:24:59,640 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:24:59,641 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c285b8b061af9a7de1e4f817de4f031: Waiting for close lock at 1731212699640Disabling compacts and flushes for region at 1731212699640Disabling writes for close at 1731212699640Writing region close event to WAL at 1731212699640Closed at 1731212699640 2024-11-10T04:24:59,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:24:59,642 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731212699642"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212699642"}]},"ts":"1731212699642"} 2024-11-10T04:24:59,645 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:24:59,646 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:24:59,646 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212699646"}]},"ts":"1731212699646"} 2024-11-10T04:24:59,648 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-10T04:24:59,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, ASSIGN}] 2024-11-10T04:24:59,650 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, ASSIGN 2024-11-10T04:24:59,651 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,38277,1731212698789; forceNewPlan=false, retain=false 2024-11-10T04:24:59,802 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c285b8b061af9a7de1e4f817de4f031, regionState=OPENING, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, ASSIGN because future has completed 2024-11-10T04:24:59,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789}] 2024-11-10T04:24:59,961 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:24:59,962 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c285b8b061af9a7de1e4f817de4f031, NAME => 'TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:24:59,962 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,962 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:24:59,962 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,962 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,963 INFO [StoreOpener-4c285b8b061af9a7de1e4f817de4f031-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,965 INFO [StoreOpener-4c285b8b061af9a7de1e4f817de4f031-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c285b8b061af9a7de1e4f817de4f031 columnFamilyName info 2024-11-10T04:24:59,965 DEBUG [StoreOpener-4c285b8b061af9a7de1e4f817de4f031-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:24:59,965 INFO [StoreOpener-4c285b8b061af9a7de1e4f817de4f031-1 {}] regionserver.HStore(327): Store=4c285b8b061af9a7de1e4f817de4f031/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:24:59,965 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,966 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,966 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,967 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,967 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,968 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,970 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:24:59,970 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4c285b8b061af9a7de1e4f817de4f031; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795591, jitterRate=0.011646538972854614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:24:59,970 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:24:59,971 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4c285b8b061af9a7de1e4f817de4f031: Running coprocessor pre-open hook at 1731212699962Writing region info on filesystem at 1731212699962Initializing all the Stores at 1731212699963 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212699963Cleaning up temporary data from old regions at 1731212699967 (+4 ms)Running coprocessor post-open hooks at 1731212699970 (+3 ms)Region opened successfully at 1731212699971 (+1 ms) 2024-11-10T04:24:59,972 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., pid=6, masterSystemTime=1731212699957 2024-11-10T04:24:59,974 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:24:59,974 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:24:59,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c285b8b061af9a7de1e4f817de4f031, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:24:59,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 because future has completed 2024-11-10T04:24:59,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T04:24:59,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 in 173 msec 2024-11-10T04:24:59,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T04:24:59,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, ASSIGN in 332 msec 2024-11-10T04:24:59,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:24:59,985 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212699984"}]},"ts":"1731212699984"} 2024-11-10T04:24:59,987 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-10T04:24:59,988 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:24:59,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 372 msec 2024-11-10T04:25:00,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:00,279 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:01,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:01,280 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:02,246 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:02,280 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:02,593 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,594 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,594 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,594 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,594 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,594 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,595 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,595 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,619 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,621 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,621 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,625 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,625 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,625 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:02,627 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,131 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:25:03,133 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,133 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,133 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,133 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,133 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,153 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,153 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,154 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,154 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,154 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,154 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,157 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,157 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,158 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,160 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:03,246 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:03,281 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:04,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:04,281 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:05,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:25:05,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-10T04:25:05,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:05,282 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:05,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-10T04:25:05,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T04:25:05,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T04:25:06,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:06,282 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:07,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:07,283 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:08,249 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:08,283 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:09,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:09,284 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:25:09,648 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-10T04:25:09,648 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-10T04:25:09,651 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-10T04:25:09,651 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:25:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2] 2024-11-10T04:25:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:09,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:25:09,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/9b2802acc0004daba44e41e71b6ba62e is 1080, key is row0001/info:/1731212709654/Put/seqid=0 2024-11-10T04:25:09,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741837_1013 (size=12509) 2024-11-10T04:25:09,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741837_1013 (size=12509) 2024-11-10T04:25:09,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/9b2802acc0004daba44e41e71b6ba62e 2024-11-10T04:25:09,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/9b2802acc0004daba44e41e71b6ba62e as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/9b2802acc0004daba44e41e71b6ba62e 2024-11-10T04:25:09,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:25:09,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/9b2802acc0004daba44e41e71b6ba62e, entries=7, sequenceid=11, filesize=12.2 K 2024-11-10T04:25:09,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 4c285b8b061af9a7de1e4f817de4f031 in 40ms, sequenceid=11, compaction requested=false 2024-11-10T04:25:09,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:53882 deadline: 1731212719702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:25:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:25:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:25:10,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:10,284 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:11,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:11,285 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:11,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:25:11,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,461 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,481 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,481 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,481 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,482 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,482 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,482 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,485 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,485 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,485 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:11,487 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:12,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:12,285 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:13,252 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:13,286 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:14,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:14,286 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:15,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:15,287 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:16,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:16,287 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:17,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:17,288 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:18,255 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:18,288 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:19,256 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:19,289 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:19,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-11-10T04:25:19,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/d14b57dc5ffd4f1980405a92d9a1ceed is 1080, key is row0008/info:/1731212709666/Put/seqid=0 2024-11-10T04:25:19,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741838_1014 (size=29761) 2024-11-10T04:25:19,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741838_1014 (size=29761) 2024-11-10T04:25:19,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/d14b57dc5ffd4f1980405a92d9a1ceed 2024-11-10T04:25:19,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/d14b57dc5ffd4f1980405a92d9a1ceed as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed 2024-11-10T04:25:19,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed, entries=23, sequenceid=37, filesize=29.1 K 2024-11-10T04:25:19,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 4c285b8b061af9a7de1e4f817de4f031 in 22ms, sequenceid=37, compaction requested=false 2024-11-10T04:25:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K 2024-11-10T04:25:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed because midkey is the same as first or last row 2024-11-10T04:25:20,256 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:20,289 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:21,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:21,290 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:21,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:25:21,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/b3b6aaf462e143e09d46dfd4f01caa47 is 1080, key is row0031/info:/1731212719780/Put/seqid=0 2024-11-10T04:25:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741839_1015 (size=12509) 2024-11-10T04:25:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741839_1015 (size=12509) 2024-11-10T04:25:21,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/b3b6aaf462e143e09d46dfd4f01caa47 2024-11-10T04:25:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/b3b6aaf462e143e09d46dfd4f01caa47 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b3b6aaf462e143e09d46dfd4f01caa47 2024-11-10T04:25:21,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b3b6aaf462e143e09d46dfd4f01caa47, entries=7, sequenceid=47, filesize=12.2 K 2024-11-10T04:25:21,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 4c285b8b061af9a7de1e4f817de4f031 in 23ms, sequenceid=47, compaction requested=true 2024-11-10T04:25:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K 2024-11-10T04:25:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed because midkey is the same as first or last row 2024-11-10T04:25:21,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c285b8b061af9a7de1e4f817de4f031:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:21,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:21,815 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:21,816 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:21,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:21,816 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): 4c285b8b061af9a7de1e4f817de4f031/info is initiating minor compaction (all files) 2024-11-10T04:25:21,816 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c285b8b061af9a7de1e4f817de4f031/info in TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:25:21,816 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/9b2802acc0004daba44e41e71b6ba62e, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b3b6aaf462e143e09d46dfd4f01caa47] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp, totalSize=53.5 K 2024-11-10T04:25:21,817 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b2802acc0004daba44e41e71b6ba62e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731212709654 2024-11-10T04:25:21,817 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14b57dc5ffd4f1980405a92d9a1ceed, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731212709666 2024-11-10T04:25:21,817 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b6aaf462e143e09d46dfd4f01caa47, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731212719780 2024-11-10T04:25:21,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/36809170cafd40d0b1722f003a8b95a5 is 1080, key is row0038/info:/1731212721792/Put/seqid=0 2024-11-10T04:25:21,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741840_1016 (size=20064) 2024-11-10T04:25:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741840_1016 (size=20064) 2024-11-10T04:25:21,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/36809170cafd40d0b1722f003a8b95a5 2024-11-10T04:25:21,835 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c285b8b061af9a7de1e4f817de4f031#info#compaction#60 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:21,836 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/b88a2b44003948f5ae6245d6ae448f05 is 1080, key is row0001/info:/1731212709654/Put/seqid=0 2024-11-10T04:25:21,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/36809170cafd40d0b1722f003a8b95a5 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/36809170cafd40d0b1722f003a8b95a5 2024-11-10T04:25:21,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741841_1017 (size=44978) 2024-11-10T04:25:21,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741841_1017 (size=44978) 2024-11-10T04:25:21,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/36809170cafd40d0b1722f003a8b95a5, entries=14, sequenceid=64, filesize=19.6 K 2024-11-10T04:25:21,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 4c285b8b061af9a7de1e4f817de4f031 in 30ms, sequenceid=64, compaction requested=false 2024-11-10T04:25:21,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:21,847 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=73.1 K, sizeToCheck=16.0 K 2024-11-10T04:25:21,847 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:21,847 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed because midkey is the same as first or last row 2024-11-10T04:25:21,851 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/b88a2b44003948f5ae6245d6ae448f05 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 2024-11-10T04:25:21,857 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c285b8b061af9a7de1e4f817de4f031/info of 4c285b8b061af9a7de1e4f817de4f031 into b88a2b44003948f5ae6245d6ae448f05(size=43.9 K), total size for store is 63.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:21,857 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., storeName=4c285b8b061af9a7de1e4f817de4f031/info, priority=13, startTime=1731212721814; duration=0sec 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 because midkey is the same as first or last row 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 because midkey is the same as first or last row 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 because midkey is the same as first or last row 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:21,857 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c285b8b061af9a7de1e4f817de4f031:info 2024-11-10T04:25:22,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:22,290 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:23,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:23,291 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:23,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:23,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/ecad47a4f53f4022959778737a91e098 is 1080, key is row0052/info:/1731212721817/Put/seqid=0 2024-11-10T04:25:23,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741842_1018 (size=20064) 2024-11-10T04:25:23,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741842_1018 (size=20064) 2024-11-10T04:25:23,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/ecad47a4f53f4022959778737a91e098 2024-11-10T04:25:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/ecad47a4f53f4022959778737a91e098 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/ecad47a4f53f4022959778737a91e098 2024-11-10T04:25:23,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/ecad47a4f53f4022959778737a91e098, entries=14, sequenceid=82, filesize=19.6 K 2024-11-10T04:25:23,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 4c285b8b061af9a7de1e4f817de4f031 in 23ms, sequenceid=82, compaction requested=true 2024-11-10T04:25:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 because midkey is the same as first or last row 2024-11-10T04:25:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c285b8b061af9a7de1e4f817de4f031:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:23,866 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:23,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:23,867 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:23,867 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): 4c285b8b061af9a7de1e4f817de4f031/info is initiating minor compaction (all files) 2024-11-10T04:25:23,868 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c285b8b061af9a7de1e4f817de4f031/info in TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:25:23,868 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/36809170cafd40d0b1722f003a8b95a5, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/ecad47a4f53f4022959778737a91e098] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp, totalSize=83.1 K 2024-11-10T04:25:23,868 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting b88a2b44003948f5ae6245d6ae448f05, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731212709654 2024-11-10T04:25:23,869 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36809170cafd40d0b1722f003a8b95a5, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731212721792 2024-11-10T04:25:23,869 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecad47a4f53f4022959778737a91e098, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731212721817 2024-11-10T04:25:23,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/33a13a8dc1064a8395571b696265c5bf is 1080, key is row0066/info:/1731212723845/Put/seqid=0 2024-11-10T04:25:23,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741843_1019 (size=20064) 2024-11-10T04:25:23,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741843_1019 (size=20064) 2024-11-10T04:25:23,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/33a13a8dc1064a8395571b696265c5bf 2024-11-10T04:25:23,882 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c285b8b061af9a7de1e4f817de4f031#info#compaction#63 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:23,882 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/3890706b877b49fa88a9131260dae409 is 1080, key is row0001/info:/1731212709654/Put/seqid=0 2024-11-10T04:25:23,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/33a13a8dc1064a8395571b696265c5bf as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/33a13a8dc1064a8395571b696265c5bf 2024-11-10T04:25:23,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741844_1020 (size=75378) 2024-11-10T04:25:23,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741844_1020 (size=75378) 2024-11-10T04:25:23,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/33a13a8dc1064a8395571b696265c5bf, entries=14, sequenceid=99, filesize=19.6 K 2024-11-10T04:25:23,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for 4c285b8b061af9a7de1e4f817de4f031 in 23ms, sequenceid=99, compaction requested=false 2024-11-10T04:25:23,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.7 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 because midkey is the same as first or last row 2024-11-10T04:25:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:23,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-10T04:25:23,894 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/3890706b877b49fa88a9131260dae409 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409 2024-11-10T04:25:23,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/2911a4caf4dc4114ad274d77990e2273 is 1080, key is row0080/info:/1731212723868/Put/seqid=0 2024-11-10T04:25:23,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741845_1021 (size=18987) 2024-11-10T04:25:23,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741845_1021 (size=18987) 2024-11-10T04:25:23,901 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c285b8b061af9a7de1e4f817de4f031/info of 4c285b8b061af9a7de1e4f817de4f031 into 3890706b877b49fa88a9131260dae409(size=73.6 K), total size for store is 93.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:23,901 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:23,901 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., storeName=4c285b8b061af9a7de1e4f817de4f031/info, priority=13, startTime=1731212723866; duration=0sec 2024-11-10T04:25:23,902 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,902 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/2911a4caf4dc4114ad274d77990e2273 2024-11-10T04:25:23,902 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,902 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,902 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,902 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,903 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:23,903 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:23,903 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c285b8b061af9a7de1e4f817de4f031:info 2024-11-10T04:25:23,904 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] assignment.AssignmentManager(1363): Split request from d1d9e3766cb1,38277,1731212698789, parent={ENCODED => 4c285b8b061af9a7de1e4f817de4f031, NAME => 'TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-10T04:25:23,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/2911a4caf4dc4114ad274d77990e2273 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/2911a4caf4dc4114ad274d77990e2273 2024-11-10T04:25:23,909 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:23,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/2911a4caf4dc4114ad274d77990e2273, entries=13, sequenceid=115, filesize=18.5 K 2024-11-10T04:25:23,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=4.20 KB/4304 for 4c285b8b061af9a7de1e4f817de4f031 in 22ms, sequenceid=115, compaction requested=true 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c285b8b061af9a7de1e4f817de4f031: 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=111.7 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=111.7 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=111.7 K, sizeToCheck=16.0 K 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T04:25:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=1 2024-11-10T04:25:23,913 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=265dc97e0d16e22599da5e120f5b06c6, daughterB=acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:23,914 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=265dc97e0d16e22599da5e120f5b06c6, daughterB=acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:23,914 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=265dc97e0d16e22599da5e120f5b06c6, daughterB=acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:23,914 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=265dc97e0d16e22599da5e120f5b06c6, daughterB=acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:23,915 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] assignment.AssignmentManager(1363): Split request from d1d9e3766cb1,38277,1731212698789, parent={ENCODED => 4c285b8b061af9a7de1e4f817de4f031, NAME => 'TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-10T04:25:23,916 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=SPLITTING, location=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:23,917 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40483 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=8dbff5332e814deb403ddad3929b3d5a, daughterB=3d8000709d1c7031ebd57fe4e4a15b08 2024-11-10T04:25:23,917 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(842): Waiting on xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=8dbff5332e814deb403ddad3929b3d5a, daughterB=3d8000709d1c7031ebd57fe4e4a15b08 held by pid=7 2024-11-10T04:25:23,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, UNASSIGN}] 2024-11-10T04:25:23,924 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, UNASSIGN 2024-11-10T04:25:23,924 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(162): LOCK_EVENT_WAIT SchemaLocking[serverLocks={},namespaceLocks={hbase=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},tableLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},regionLocks={},peerLocks={},metaLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},globalLocks={}] 2024-11-10T04:25:23,924 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=8dbff5332e814deb403ddad3929b3d5a, daughterB=3d8000709d1c7031ebd57fe4e4a15b08 2024-11-10T04:25:23,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=4c285b8b061af9a7de1e4f817de4f031, regionState=CLOSING, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:23,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, UNASSIGN because future has completed 2024-11-10T04:25:23,928 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-10T04:25:23,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789}] 2024-11-10T04:25:24,086 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,086 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-10T04:25:24,086 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 4c285b8b061af9a7de1e4f817de4f031, disabling compactions & flushes 2024-11-10T04:25:24,087 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:25:24,087 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:25:24,087 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. after waiting 0 ms 2024-11-10T04:25:24,087 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:25:24,087 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(2902): Flushing 4c285b8b061af9a7de1e4f817de4f031 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-11-10T04:25:24,091 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/34e0bb4e8cf543d0ad146e3800c8f62a is 1080, key is row0093/info:/1731212723893/Put/seqid=0 2024-11-10T04:25:24,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741846_1022 (size=9270) 2024-11-10T04:25:24,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741846_1022 (size=9270) 2024-11-10T04:25:24,097 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/34e0bb4e8cf543d0ad146e3800c8f62a 2024-11-10T04:25:24,102 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/.tmp/info/34e0bb4e8cf543d0ad146e3800c8f62a as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/34e0bb4e8cf543d0ad146e3800c8f62a 2024-11-10T04:25:24,107 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/34e0bb4e8cf543d0ad146e3800c8f62a, entries=4, sequenceid=123, filesize=9.1 K 2024-11-10T04:25:24,108 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 4c285b8b061af9a7de1e4f817de4f031 in 21ms, sequenceid=123, compaction requested=true 2024-11-10T04:25:24,109 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/9b2802acc0004daba44e41e71b6ba62e, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b3b6aaf462e143e09d46dfd4f01caa47, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/36809170cafd40d0b1722f003a8b95a5, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/ecad47a4f53f4022959778737a91e098] to archive 2024-11-10T04:25:24,109 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:25:24,111 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/9b2802acc0004daba44e41e71b6ba62e to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/9b2802acc0004daba44e41e71b6ba62e 2024-11-10T04:25:24,112 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/d14b57dc5ffd4f1980405a92d9a1ceed 2024-11-10T04:25:24,114 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b88a2b44003948f5ae6245d6ae448f05 2024-11-10T04:25:24,115 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b3b6aaf462e143e09d46dfd4f01caa47 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/b3b6aaf462e143e09d46dfd4f01caa47 2024-11-10T04:25:24,116 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/36809170cafd40d0b1722f003a8b95a5 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/36809170cafd40d0b1722f003a8b95a5 2024-11-10T04:25:24,117 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/ecad47a4f53f4022959778737a91e098 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/ecad47a4f53f4022959778737a91e098 2024-11-10T04:25:24,123 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=1 2024-11-10T04:25:24,124 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. 2024-11-10T04:25:24,124 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 4c285b8b061af9a7de1e4f817de4f031: Waiting for close lock at 1731212724086Running coprocessor pre-close hooks at 1731212724086Disabling compacts and flushes for region at 1731212724086Disabling writes for close at 1731212724087 (+1 ms)Obtaining lock to block concurrent updates at 1731212724087Preparing flush snapshotting stores in 4c285b8b061af9a7de1e4f817de4f031 at 1731212724087Finished memstore snapshotting TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., syncing WAL and waiting on mvcc, flushsize=dataSize=4304, getHeapSize=4848, getOffHeapSize=0, getCellsCount=4 at 1731212724087Flushing stores of TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. at 1731212724088 (+1 ms)Flushing 4c285b8b061af9a7de1e4f817de4f031/info: creating writer at 1731212724088Flushing 4c285b8b061af9a7de1e4f817de4f031/info: appending metadata at 1731212724091 (+3 ms)Flushing 4c285b8b061af9a7de1e4f817de4f031/info: closing flushed file at 1731212724091Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bc943ff: reopening flushed file at 1731212724102 (+11 ms)Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 4c285b8b061af9a7de1e4f817de4f031 in 21ms, sequenceid=123, compaction requested=true at 1731212724108 (+6 ms)Writing region close event to WAL at 1731212724120 (+12 ms)Running coprocessor post-close hooks at 1731212724124 (+4 ms)Closed at 1731212724124 2024-11-10T04:25:24,126 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,127 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=4c285b8b061af9a7de1e4f817de4f031, regionState=CLOSED 2024-11-10T04:25:24,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 because future has completed 2024-11-10T04:25:24,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T04:25:24,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c285b8b061af9a7de1e4f817de4f031, server=d1d9e3766cb1,38277,1731212698789 in 202 msec 2024-11-10T04:25:24,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-10T04:25:24,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4c285b8b061af9a7de1e4f817de4f031, UNASSIGN in 211 msec 2024-11-10T04:25:24,142 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:24,146 INFO [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 4 storefiles, region=4c285b8b061af9a7de1e4f817de4f031, threads=4 2024-11-10T04:25:24,148 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/33a13a8dc1064a8395571b696265c5bf for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,148 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/34e0bb4e8cf543d0ad146e3800c8f62a for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,148 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/2911a4caf4dc4114ad274d77990e2273 for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,148 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409 for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,158 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/2911a4caf4dc4114ad274d77990e2273, top=true 2024-11-10T04:25:24,158 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/33a13a8dc1064a8395571b696265c5bf, top=true 2024-11-10T04:25:24,159 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/34e0bb4e8cf543d0ad146e3800c8f62a, top=true 2024-11-10T04:25:24,171 INFO [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273 for child: acfe3a64f0010d1915c5e283bf349529, parent: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,171 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/2911a4caf4dc4114ad274d77990e2273 for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,171 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf for child: acfe3a64f0010d1915c5e283bf349529, parent: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,171 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/33a13a8dc1064a8395571b696265c5bf for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,173 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a for child: acfe3a64f0010d1915c5e283bf349529, parent: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,173 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/34e0bb4e8cf543d0ad146e3800c8f62a for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741847_1023 (size=27) 2024-11-10T04:25:24,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741847_1023 (size=27) 2024-11-10T04:25:24,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741848_1024 (size=27) 2024-11-10T04:25:24,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741848_1024 (size=27) 2024-11-10T04:25:24,185 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409 for region: 4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:24,187 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 4c285b8b061af9a7de1e4f817de4f031 Daughter A: [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031] storefiles, Daughter B: [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031] storefiles. 2024-11-10T04:25:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741849_1025 (size=71) 2024-11-10T04:25:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741849_1025 (size=71) 2024-11-10T04:25:24,197 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:24,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741850_1026 (size=71) 2024-11-10T04:25:24,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741850_1026 (size=71) 2024-11-10T04:25:24,210 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:24,220 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1 2024-11-10T04:25:24,222 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1 2024-11-10T04:25:24,224 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731212724224"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731212724224"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731212724224"}]},"ts":"1731212724224"} 2024-11-10T04:25:24,224 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731212724224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212724224"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731212724224"}]},"ts":"1731212724224"} 2024-11-10T04:25:24,225 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731212724224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212724224"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731212724224"}]},"ts":"1731212724224"} 2024-11-10T04:25:24,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=265dc97e0d16e22599da5e120f5b06c6, ASSIGN}, {pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=acfe3a64f0010d1915c5e283bf349529, ASSIGN}] 2024-11-10T04:25:24,245 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=acfe3a64f0010d1915c5e283bf349529, ASSIGN 2024-11-10T04:25:24,245 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=265dc97e0d16e22599da5e120f5b06c6, ASSIGN 2024-11-10T04:25:24,246 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=acfe3a64f0010d1915c5e283bf349529, ASSIGN; state=SPLITTING_NEW, location=d1d9e3766cb1,38277,1731212698789; forceNewPlan=false, retain=false 2024-11-10T04:25:24,246 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=265dc97e0d16e22599da5e120f5b06c6, ASSIGN; state=SPLITTING_NEW, location=d1d9e3766cb1,38277,1731212698789; forceNewPlan=false, retain=false 2024-11-10T04:25:24,259 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:24,291 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:24,397 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=acfe3a64f0010d1915c5e283bf349529, regionState=OPENING, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:24,397 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=265dc97e0d16e22599da5e120f5b06c6, regionState=OPENING, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:24,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=265dc97e0d16e22599da5e120f5b06c6, ASSIGN because future has completed 2024-11-10T04:25:24,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 265dc97e0d16e22599da5e120f5b06c6, server=d1d9e3766cb1,38277,1731212698789}] 2024-11-10T04:25:24,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=acfe3a64f0010d1915c5e283bf349529, ASSIGN because future has completed 2024-11-10T04:25:24,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure acfe3a64f0010d1915c5e283bf349529, server=d1d9e3766cb1,38277,1731212698789}] 2024-11-10T04:25:24,556 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:24,557 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 265dc97e0d16e22599da5e120f5b06c6, NAME => 'TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.', STARTKEY => '', ENDKEY => 'row0062'} 2024-11-10T04:25:24,557 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,557 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:25:24,557 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,557 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,559 INFO [StoreOpener-265dc97e0d16e22599da5e120f5b06c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,559 INFO [StoreOpener-265dc97e0d16e22599da5e120f5b06c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 265dc97e0d16e22599da5e120f5b06c6 columnFamilyName info 2024-11-10T04:25:24,560 DEBUG [StoreOpener-265dc97e0d16e22599da5e120f5b06c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:24,571 DEBUG [StoreOpener-265dc97e0d16e22599da5e120f5b06c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031->hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409-bottom 2024-11-10T04:25:24,572 INFO [StoreOpener-265dc97e0d16e22599da5e120f5b06c6-1 {}] regionserver.HStore(327): Store=265dc97e0d16e22599da5e120f5b06c6/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:25:24,572 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,573 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,574 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,574 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,574 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,576 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,577 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 265dc97e0d16e22599da5e120f5b06c6; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=864328, jitterRate=0.09905010461807251}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:25:24,577 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:24,577 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 265dc97e0d16e22599da5e120f5b06c6: Running coprocessor pre-open hook at 1731212724557Writing region info on filesystem at 1731212724557Initializing all the Stores at 1731212724558 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212724558Cleaning up temporary data from old regions at 1731212724574 (+16 ms)Running coprocessor post-open hooks at 1731212724577 (+3 ms)Region opened successfully at 1731212724577 2024-11-10T04:25:24,578 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6., pid=13, masterSystemTime=1731212724553 2024-11-10T04:25:24,578 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 265dc97e0d16e22599da5e120f5b06c6:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:24,578 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:24,578 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-10T04:25:24,579 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:24,579 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): 265dc97e0d16e22599da5e120f5b06c6/info is initiating minor compaction (all files) 2024-11-10T04:25:24,579 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 265dc97e0d16e22599da5e120f5b06c6/info in TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:24,579 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031->hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409-bottom] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/.tmp, totalSize=73.6 K 2024-11-10T04:25:24,580 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731212709654 2024-11-10T04:25:24,581 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:24,581 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:24,581 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:24,581 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7752): Opening region: {ENCODED => acfe3a64f0010d1915c5e283bf349529, NAME => 'TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.', STARTKEY => 'row0062', ENDKEY => ''} 2024-11-10T04:25:24,581 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,581 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:25:24,581 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7794): checking encryption for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,581 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7797): checking classloading for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,581 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=265dc97e0d16e22599da5e120f5b06c6, regionState=OPEN, openSeqNum=127, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:24,583 INFO [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,583 INFO [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acfe3a64f0010d1915c5e283bf349529 columnFamilyName info 2024-11-10T04:25:24,584 DEBUG [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:24,584 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-10T04:25:24,584 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:25:24,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB 2024-11-10T04:25:24,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 265dc97e0d16e22599da5e120f5b06c6, server=d1d9e3766cb1,38277,1731212698789 because future has completed 2024-11-10T04:25:24,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11 2024-11-10T04:25:24,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 265dc97e0d16e22599da5e120f5b06c6, server=d1d9e3766cb1,38277,1731212698789 in 185 msec 2024-11-10T04:25:24,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=265dc97e0d16e22599da5e120f5b06c6, ASSIGN in 346 msec 2024-11-10T04:25:24,593 DEBUG [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031->hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409-top 2024-11-10T04:25:24,598 DEBUG [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273 2024-11-10T04:25:24,602 DEBUG [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf 2024-11-10T04:25:24,604 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 265dc97e0d16e22599da5e120f5b06c6#info#compaction#66 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:24,605 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/.tmp/info/1823945b6a52428ab08ae1585f4a605f is 1080, key is row0001/info:/1731212709654/Put/seqid=0 2024-11-10T04:25:24,605 DEBUG [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a 2024-11-10T04:25:24,605 INFO [StoreOpener-acfe3a64f0010d1915c5e283bf349529-1 {}] regionserver.HStore(327): Store=acfe3a64f0010d1915c5e283bf349529/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:25:24,606 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1038): replaying wal for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/info/cf7326328c6c4faabd4897f075860ea5 is 193, key is TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529./info:regioninfo/1731212724397/Put/seqid=0 2024-11-10T04:25:24,606 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,608 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,608 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1048): stopping wal replay for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,608 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1060): Cleaning up temporary data for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741851_1027 (size=70862) 2024-11-10T04:25:24,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741851_1027 (size=70862) 2024-11-10T04:25:24,610 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1093): writing seq id for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,611 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1114): Opened acfe3a64f0010d1915c5e283bf349529; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=708345, jitterRate=-0.09929348528385162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:25:24,611 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1122): Running coprocessor post-open hooks for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:24,612 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1006): Region open journal for acfe3a64f0010d1915c5e283bf349529: Running coprocessor pre-open hook at 1731212724581Writing region info on filesystem at 1731212724581Initializing all the Stores at 1731212724582 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212724582Cleaning up temporary data from old regions at 1731212724608 (+26 ms)Running coprocessor post-open hooks at 1731212724611 (+3 ms)Region opened successfully at 1731212724611 2024-11-10T04:25:24,613 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., pid=14, masterSystemTime=1731212724553 2024-11-10T04:25:24,613 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(403): Add compact mark for store acfe3a64f0010d1915c5e283bf349529:info, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:25:24,613 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:24,613 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:25:24,615 INFO [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:24,615 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.HStore(1541): acfe3a64f0010d1915c5e283bf349529/info is initiating minor compaction (all files) 2024-11-10T04:25:24,615 INFO [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acfe3a64f0010d1915c5e283bf349529/info in TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:24,615 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:24,615 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:24,615 INFO [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031->hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409-top, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp, totalSize=120.8 K 2024-11-10T04:25:24,616 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/.tmp/info/1823945b6a52428ab08ae1585f4a605f as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/info/1823945b6a52428ab08ae1585f4a605f 2024-11-10T04:25:24,616 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] compactions.Compactor(225): Compacting 3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731212709654 2024-11-10T04:25:24,616 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=acfe3a64f0010d1915c5e283bf349529, regionState=OPEN, openSeqNum=127, regionLocation=d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:24,616 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731212723845 2024-11-10T04:25:24,617 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731212723868 2024-11-10T04:25:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741852_1028 (size=9847) 2024-11-10T04:25:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741852_1028 (size=9847) 2024-11-10T04:25:24,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/info/cf7326328c6c4faabd4897f075860ea5 2024-11-10T04:25:24,618 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a, keycount=4, bloomtype=ROW, size=9.1 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731212723893 2024-11-10T04:25:24,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure acfe3a64f0010d1915c5e283bf349529, server=d1d9e3766cb1,38277,1731212698789 because future has completed 2024-11-10T04:25:24,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=12 2024-11-10T04:25:24,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure acfe3a64f0010d1915c5e283bf349529, server=d1d9e3766cb1,38277,1731212698789 in 219 msec 2024-11-10T04:25:24,624 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 265dc97e0d16e22599da5e120f5b06c6/info of 265dc97e0d16e22599da5e120f5b06c6 into 1823945b6a52428ab08ae1585f4a605f(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:24,625 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 265dc97e0d16e22599da5e120f5b06c6: 2024-11-10T04:25:24,625 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6., storeName=265dc97e0d16e22599da5e120f5b06c6/info, priority=15, startTime=1731212724578; duration=0sec 2024-11-10T04:25:24,625 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:24,625 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 265dc97e0d16e22599da5e120f5b06c6:info 2024-11-10T04:25:24,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=7 2024-11-10T04:25:24,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=acfe3a64f0010d1915c5e283bf349529, ASSIGN in 380 msec 2024-11-10T04:25:24,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=265dc97e0d16e22599da5e120f5b06c6, daughterB=acfe3a64f0010d1915c5e283bf349529 in 717 msec 2024-11-10T04:25:24,629 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=8dbff5332e814deb403ddad3929b3d5a, daughterB=3d8000709d1c7031ebd57fe4e4a15b08 2024-11-10T04:25:24,629 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=8dbff5332e814deb403ddad3929b3d5a, daughterB=3d8000709d1c7031ebd57fe4e4a15b08 2024-11-10T04:25:24,630 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=8dbff5332e814deb403ddad3929b3d5a, daughterB=3d8000709d1c7031ebd57fe4e4a15b08 2024-11-10T04:25:24,630 INFO [PEWorker-1 {}] assignment.SplitTableRegionProcedure(534): Split of {ENCODED => 4c285b8b061af9a7de1e4f817de4f031, NAME => 'TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031.', STARTKEY => '', ENDKEY => ''} skipped; state is already SPLIT 2024-11-10T04:25:24,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4c285b8b061af9a7de1e4f817de4f031, daughterA=8dbff5332e814deb403ddad3929b3d5a, daughterB=3d8000709d1c7031ebd57fe4e4a15b08 in 714 msec 2024-11-10T04:25:24,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/ns/1419b30abc774e0a92a1a19e35580be8 is 43, key is default/ns:d/1731212699589/Put/seqid=0 2024-11-10T04:25:24,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741853_1029 (size=5153) 2024-11-10T04:25:24,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741853_1029 (size=5153) 2024-11-10T04:25:24,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/ns/1419b30abc774e0a92a1a19e35580be8 2024-11-10T04:25:24,650 INFO [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): acfe3a64f0010d1915c5e283bf349529#info#compaction#69 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:24,651 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/c044c89d18484b57a1a4b9a853478321 is 1080, key is row0062/info:/1731212721836/Put/seqid=0 2024-11-10T04:25:24,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741854_1030 (size=43081) 2024-11-10T04:25:24,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741854_1030 (size=43081) 2024-11-10T04:25:24,663 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/c044c89d18484b57a1a4b9a853478321 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/c044c89d18484b57a1a4b9a853478321 2024-11-10T04:25:24,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/table/7a67a2e399bd428b9928732406aaeca4 is 65, key is TestLogRolling-testLogRolling/table:state/1731212699984/Put/seqid=0 2024-11-10T04:25:24,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741855_1031 (size=5340) 2024-11-10T04:25:24,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741855_1031 (size=5340) 2024-11-10T04:25:24,670 INFO [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in acfe3a64f0010d1915c5e283bf349529/info of acfe3a64f0010d1915c5e283bf349529 into c044c89d18484b57a1a4b9a853478321(size=42.1 K), total size for store is 42.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:24,670 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:24,670 INFO [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., storeName=acfe3a64f0010d1915c5e283bf349529/info, priority=12, startTime=1731212724613; duration=0sec 2024-11-10T04:25:24,670 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:24,670 DEBUG [RS:0;d1d9e3766cb1:38277-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acfe3a64f0010d1915c5e283bf349529:info 2024-11-10T04:25:24,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/table/7a67a2e399bd428b9928732406aaeca4 2024-11-10T04:25:24,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/info/cf7326328c6c4faabd4897f075860ea5 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/info/cf7326328c6c4faabd4897f075860ea5 2024-11-10T04:25:24,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/info/cf7326328c6c4faabd4897f075860ea5, entries=30, sequenceid=17, filesize=9.6 K 2024-11-10T04:25:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/ns/1419b30abc774e0a92a1a19e35580be8 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/ns/1419b30abc774e0a92a1a19e35580be8 2024-11-10T04:25:24,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/ns/1419b30abc774e0a92a1a19e35580be8, entries=2, sequenceid=17, filesize=5.0 K 2024-11-10T04:25:24,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/table/7a67a2e399bd428b9928732406aaeca4 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/table/7a67a2e399bd428b9928732406aaeca4 2024-11-10T04:25:24,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/table/7a67a2e399bd428b9928732406aaeca4, entries=2, sequenceid=17, filesize=5.2 K 2024-11-10T04:25:24,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 110ms, sequenceid=17, compaction requested=false 2024-11-10T04:25:24,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T04:25:25,259 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:25,292 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:53882 deadline: 1731212735901, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. is not online on d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. is not online on d1d9e3766cb1,38277,1731212698789 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:25:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031. is not online on d1d9e3766cb1,38277,1731212698789 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:25:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731212699615.4c285b8b061af9a7de1e4f817de4f031., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=2 from cache 2024-11-10T04:25:26,260 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:26,292 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:27,260 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:27,293 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:28,261 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:28,293 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:28,728 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:25:29,125 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,125 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,125 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,125 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,125 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,126 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,126 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,126 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,145 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,146 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,146 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,146 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,146 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,146 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,149 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,149 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,149 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,151 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,261 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:29,293 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:29,658 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T04:25:29,659 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,659 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,660 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,660 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,660 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,660 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,661 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,661 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,683 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,683 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,683 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,683 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,686 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,686 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,687 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:29,689 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T04:25:30,262 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:30,294 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:31,263 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:31,294 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:32,263 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:32,295 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:33,264 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:33,295 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:34,264 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:34,296 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:35,265 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:35,296 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., hostname=d1d9e3766cb1,38277,1731212698789, seqNum=127] 2024-11-10T04:25:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:35,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:25:35,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/25d20b61a5244f4bb02956413e67e4bd is 1080, key is row0097/info:/1731212735984/Put/seqid=0 2024-11-10T04:25:36,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741856_1032 (size=12516) 2024-11-10T04:25:36,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741856_1032 (size=12516) 2024-11-10T04:25:36,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/25d20b61a5244f4bb02956413e67e4bd 2024-11-10T04:25:36,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/25d20b61a5244f4bb02956413e67e4bd as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/25d20b61a5244f4bb02956413e67e4bd 2024-11-10T04:25:36,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/25d20b61a5244f4bb02956413e67e4bd, entries=7, sequenceid=137, filesize=12.2 K 2024-11-10T04:25:36,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for acfe3a64f0010d1915c5e283bf349529 in 44ms, sequenceid=137, compaction requested=false 2024-11-10T04:25:36,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:36,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:36,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/5c09b10b45704e8c87b58eb6989416f7 is 1080, key is row0104/info:/1731212735996/Put/seqid=0 2024-11-10T04:25:36,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741857_1033 (size=20078) 2024-11-10T04:25:36,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741857_1033 (size=20078) 2024-11-10T04:25:36,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/5c09b10b45704e8c87b58eb6989416f7 2024-11-10T04:25:36,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/5c09b10b45704e8c87b58eb6989416f7 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5c09b10b45704e8c87b58eb6989416f7 2024-11-10T04:25:36,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5c09b10b45704e8c87b58eb6989416f7, entries=14, sequenceid=154, filesize=19.6 K 2024-11-10T04:25:36,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for acfe3a64f0010d1915c5e283bf349529 in 39ms, sequenceid=154, compaction requested=true 2024-11-10T04:25:36,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acfe3a64f0010d1915c5e283bf349529:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:36,080 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:36,081 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 75675 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:36,081 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): acfe3a64f0010d1915c5e283bf349529/info is initiating minor compaction (all files) 2024-11-10T04:25:36,081 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acfe3a64f0010d1915c5e283bf349529/info in TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:36,081 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/c044c89d18484b57a1a4b9a853478321, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/25d20b61a5244f4bb02956413e67e4bd, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5c09b10b45704e8c87b58eb6989416f7] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp, totalSize=73.9 K 2024-11-10T04:25:36,082 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting c044c89d18484b57a1a4b9a853478321, keycount=35, bloomtype=ROW, size=42.1 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731212721836 2024-11-10T04:25:36,082 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25d20b61a5244f4bb02956413e67e4bd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731212735984 2024-11-10T04:25:36,082 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c09b10b45704e8c87b58eb6989416f7, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731212735996 2024-11-10T04:25:36,095 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acfe3a64f0010d1915c5e283bf349529#info#compaction#73 average throughput is 57.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:36,095 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/62f87626d6e7427d91c7990fb3620bb2 is 1080, key is row0062/info:/1731212721836/Put/seqid=0 2024-11-10T04:25:36,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741858_1034 (size=65889) 2024-11-10T04:25:36,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741858_1034 (size=65889) 2024-11-10T04:25:36,107 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/62f87626d6e7427d91c7990fb3620bb2 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/62f87626d6e7427d91c7990fb3620bb2 2024-11-10T04:25:36,112 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acfe3a64f0010d1915c5e283bf349529/info of acfe3a64f0010d1915c5e283bf349529 into 62f87626d6e7427d91c7990fb3620bb2(size=64.3 K), total size for store is 64.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:36,112 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:36,112 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., storeName=acfe3a64f0010d1915c5e283bf349529/info, priority=13, startTime=1731212736079; duration=0sec 2024-11-10T04:25:36,113 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:36,113 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acfe3a64f0010d1915c5e283bf349529:info 2024-11-10T04:25:36,266 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:36,297 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:37,266 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:37,297 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:38,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-10T04:25:38,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/efb8fbf85c094c5c8227ec8dc4b72673 is 1080, key is row0118/info:/1731212736042/Put/seqid=0 2024-11-10T04:25:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741859_1035 (size=17906) 2024-11-10T04:25:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741859_1035 (size=17906) 2024-11-10T04:25:38,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/efb8fbf85c094c5c8227ec8dc4b72673 2024-11-10T04:25:38,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/efb8fbf85c094c5c8227ec8dc4b72673 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/efb8fbf85c094c5c8227ec8dc4b72673 2024-11-10T04:25:38,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/efb8fbf85c094c5c8227ec8dc4b72673, entries=12, sequenceid=170, filesize=17.5 K 2024-11-10T04:25:38,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=12.61 KB/12912 for acfe3a64f0010d1915c5e283bf349529 in 21ms, sequenceid=170, compaction requested=false 2024-11-10T04:25:38,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:38,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:38,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/a96b162f51db4c77b501faeacc2a2296 is 1080, key is row0130/info:/1731212738063/Put/seqid=0 2024-11-10T04:25:38,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741860_1036 (size=20078) 2024-11-10T04:25:38,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741860_1036 (size=20078) 2024-11-10T04:25:38,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/a96b162f51db4c77b501faeacc2a2296 2024-11-10T04:25:38,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/a96b162f51db4c77b501faeacc2a2296 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/a96b162f51db4c77b501faeacc2a2296 2024-11-10T04:25:38,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/a96b162f51db4c77b501faeacc2a2296, entries=14, sequenceid=187, filesize=19.6 K 2024-11-10T04:25:38,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for acfe3a64f0010d1915c5e283bf349529 in 21ms, sequenceid=187, compaction requested=true 2024-11-10T04:25:38,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:38,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acfe3a64f0010d1915c5e283bf349529:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:38,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:38,106 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:38,107 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:38,107 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): acfe3a64f0010d1915c5e283bf349529/info is initiating minor compaction (all files) 2024-11-10T04:25:38,108 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acfe3a64f0010d1915c5e283bf349529/info in TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:38,108 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/62f87626d6e7427d91c7990fb3620bb2, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/efb8fbf85c094c5c8227ec8dc4b72673, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/a96b162f51db4c77b501faeacc2a2296] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp, totalSize=101.4 K 2024-11-10T04:25:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:38,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:38,108 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f87626d6e7427d91c7990fb3620bb2, keycount=56, bloomtype=ROW, size=64.3 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731212721836 2024-11-10T04:25:38,109 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb8fbf85c094c5c8227ec8dc4b72673, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731212736042 2024-11-10T04:25:38,109 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96b162f51db4c77b501faeacc2a2296, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731212738063 2024-11-10T04:25:38,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/5ab57b3d4ad94aacaddd064654082a36 is 1080, key is row0144/info:/1731212738086/Put/seqid=0 2024-11-10T04:25:38,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741861_1037 (size=20078) 2024-11-10T04:25:38,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741861_1037 (size=20078) 2024-11-10T04:25:38,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/5ab57b3d4ad94aacaddd064654082a36 2024-11-10T04:25:38,121 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acfe3a64f0010d1915c5e283bf349529#info#compaction#77 average throughput is 84.14 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:38,122 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/3d7a34e1b7d54e87a77fe3bff08b2edf is 1080, key is row0062/info:/1731212721836/Put/seqid=0 2024-11-10T04:25:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/5ab57b3d4ad94aacaddd064654082a36 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5ab57b3d4ad94aacaddd064654082a36 2024-11-10T04:25:38,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741862_1038 (size=94096) 2024-11-10T04:25:38,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5ab57b3d4ad94aacaddd064654082a36, entries=14, sequenceid=204, filesize=19.6 K 2024-11-10T04:25:38,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741862_1038 (size=94096) 2024-11-10T04:25:38,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=3.15 KB/3228 for acfe3a64f0010d1915c5e283bf349529 in 22ms, sequenceid=204, compaction requested=false 2024-11-10T04:25:38,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:38,267 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:38,298 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:38,535 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/3d7a34e1b7d54e87a77fe3bff08b2edf as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3d7a34e1b7d54e87a77fe3bff08b2edf 2024-11-10T04:25:38,541 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acfe3a64f0010d1915c5e283bf349529/info of acfe3a64f0010d1915c5e283bf349529 into 3d7a34e1b7d54e87a77fe3bff08b2edf(size=91.9 K), total size for store is 111.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:38,541 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:38,541 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., storeName=acfe3a64f0010d1915c5e283bf349529/info, priority=13, startTime=1731212738106; duration=0sec 2024-11-10T04:25:38,541 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:38,541 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acfe3a64f0010d1915c5e283bf349529:info 2024-11-10T04:25:39,267 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:39,298 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:39,670 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T04:25:39,670 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T04:25:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:40,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:25:40,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0c58865d234f488a9b0ac543640046d0 is 1080, key is row0158/info:/1731212738109/Put/seqid=0 2024-11-10T04:25:40,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741863_1039 (size=12516) 2024-11-10T04:25:40,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741863_1039 (size=12516) 2024-11-10T04:25:40,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0c58865d234f488a9b0ac543640046d0 2024-11-10T04:25:40,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0c58865d234f488a9b0ac543640046d0 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0c58865d234f488a9b0ac543640046d0 2024-11-10T04:25:40,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0c58865d234f488a9b0ac543640046d0, entries=7, sequenceid=215, filesize=12.2 K 2024-11-10T04:25:40,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for acfe3a64f0010d1915c5e283bf349529 in 21ms, sequenceid=215, compaction requested=true 2024-11-10T04:25:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acfe3a64f0010d1915c5e283bf349529:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:40,141 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:40,142 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 126690 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:40,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T04:25:40,143 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): acfe3a64f0010d1915c5e283bf349529/info is initiating minor compaction (all files) 2024-11-10T04:25:40,143 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acfe3a64f0010d1915c5e283bf349529/info in TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:40,143 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3d7a34e1b7d54e87a77fe3bff08b2edf, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5ab57b3d4ad94aacaddd064654082a36, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0c58865d234f488a9b0ac543640046d0] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp, totalSize=123.7 K 2024-11-10T04:25:40,143 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7a34e1b7d54e87a77fe3bff08b2edf, keycount=82, bloomtype=ROW, size=91.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731212721836 2024-11-10T04:25:40,144 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ab57b3d4ad94aacaddd064654082a36, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731212738086 2024-11-10T04:25:40,144 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c58865d234f488a9b0ac543640046d0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731212738109 2024-11-10T04:25:40,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/047ddf93e0a34b2bbb72e9c883149968 is 1080, key is row0165/info:/1731212740121/Put/seqid=0 2024-11-10T04:25:40,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741864_1040 (size=21156) 2024-11-10T04:25:40,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741864_1040 (size=21156) 2024-11-10T04:25:40,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/047ddf93e0a34b2bbb72e9c883149968 2024-11-10T04:25:40,156 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acfe3a64f0010d1915c5e283bf349529#info#compaction#80 average throughput is 52.85 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:40,157 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/8a08062c0d3a4beba7063ce5c9e2c80f is 1080, key is row0062/info:/1731212721836/Put/seqid=0 2024-11-10T04:25:40,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/047ddf93e0a34b2bbb72e9c883149968 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/047ddf93e0a34b2bbb72e9c883149968 2024-11-10T04:25:40,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741865_1041 (size=116840) 2024-11-10T04:25:40,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/047ddf93e0a34b2bbb72e9c883149968, entries=15, sequenceid=233, filesize=20.7 K 2024-11-10T04:25:40,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741865_1041 (size=116840) 2024-11-10T04:25:40,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for acfe3a64f0010d1915c5e283bf349529 in 23ms, sequenceid=233, compaction requested=false 2024-11-10T04:25:40,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:40,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-10T04:25:40,170 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/8a08062c0d3a4beba7063ce5c9e2c80f as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/8a08062c0d3a4beba7063ce5c9e2c80f 2024-11-10T04:25:40,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/77770f75b92e4c56aed2d7f0d5e8124c is 1080, key is row0180/info:/1731212740143/Put/seqid=0 2024-11-10T04:25:40,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741866_1042 (size=19000) 2024-11-10T04:25:40,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741866_1042 (size=19000) 2024-11-10T04:25:40,175 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acfe3a64f0010d1915c5e283bf349529/info of acfe3a64f0010d1915c5e283bf349529 into 8a08062c0d3a4beba7063ce5c9e2c80f(size=114.1 K), total size for store is 134.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:40,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/77770f75b92e4c56aed2d7f0d5e8124c 2024-11-10T04:25:40,176 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:40,176 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., storeName=acfe3a64f0010d1915c5e283bf349529/info, priority=13, startTime=1731212740141; duration=0sec 2024-11-10T04:25:40,176 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:40,176 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acfe3a64f0010d1915c5e283bf349529:info 2024-11-10T04:25:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/77770f75b92e4c56aed2d7f0d5e8124c as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/77770f75b92e4c56aed2d7f0d5e8124c 2024-11-10T04:25:40,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/77770f75b92e4c56aed2d7f0d5e8124c, entries=13, sequenceid=249, filesize=18.6 K 2024-11-10T04:25:40,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=0 B/0 for acfe3a64f0010d1915c5e283bf349529 in 19ms, sequenceid=249, compaction requested=true 2024-11-10T04:25:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acfe3a64f0010d1915c5e283bf349529:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:40,185 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:40,186 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 156996 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:40,186 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): acfe3a64f0010d1915c5e283bf349529/info is initiating minor compaction (all files) 2024-11-10T04:25:40,186 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acfe3a64f0010d1915c5e283bf349529/info in TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:40,186 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/8a08062c0d3a4beba7063ce5c9e2c80f, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/047ddf93e0a34b2bbb72e9c883149968, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/77770f75b92e4c56aed2d7f0d5e8124c] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp, totalSize=153.3 K 2024-11-10T04:25:40,187 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a08062c0d3a4beba7063ce5c9e2c80f, keycount=103, bloomtype=ROW, size=114.1 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731212721836 2024-11-10T04:25:40,187 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047ddf93e0a34b2bbb72e9c883149968, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731212740121 2024-11-10T04:25:40,187 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77770f75b92e4c56aed2d7f0d5e8124c, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731212740143 2024-11-10T04:25:40,197 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acfe3a64f0010d1915c5e283bf349529#info#compaction#82 average throughput is 67.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:40,198 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/4422e526273a4d10823d7a4ec484e55d is 1080, key is row0062/info:/1731212721836/Put/seqid=0 2024-11-10T04:25:40,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741867_1043 (size=147331) 2024-11-10T04:25:40,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741867_1043 (size=147331) 2024-11-10T04:25:40,208 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/4422e526273a4d10823d7a4ec484e55d as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/4422e526273a4d10823d7a4ec484e55d 2024-11-10T04:25:40,214 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acfe3a64f0010d1915c5e283bf349529/info of acfe3a64f0010d1915c5e283bf349529 into 4422e526273a4d10823d7a4ec484e55d(size=143.9 K), total size for store is 143.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:40,214 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:40,214 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., storeName=acfe3a64f0010d1915c5e283bf349529/info, priority=13, startTime=1731212740185; duration=0sec 2024-11-10T04:25:40,214 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:40,214 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acfe3a64f0010d1915c5e283bf349529:info 2024-11-10T04:25:40,268 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:40,299 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:41,269 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:41,299 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:42,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T04:25:42,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0af053fbd84440c492cfedfe1a7563fa is 1080, key is row0193/info:/1731212742167/Put/seqid=0 2024-11-10T04:25:42,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741868_1044 (size=12521) 2024-11-10T04:25:42,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741868_1044 (size=12521) 2024-11-10T04:25:42,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0af053fbd84440c492cfedfe1a7563fa 2024-11-10T04:25:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0af053fbd84440c492cfedfe1a7563fa as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0af053fbd84440c492cfedfe1a7563fa 2024-11-10T04:25:42,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0af053fbd84440c492cfedfe1a7563fa, entries=7, sequenceid=261, filesize=12.2 K 2024-11-10T04:25:42,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for acfe3a64f0010d1915c5e283bf349529 in 22ms, sequenceid=261, compaction requested=false 2024-11-10T04:25:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:42,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T04:25:42,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/bdd063820d9b4737993c3645cb9c0e0f is 1080, key is row0200/info:/1731212742179/Put/seqid=0 2024-11-10T04:25:42,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741869_1045 (size=21171) 2024-11-10T04:25:42,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741869_1045 (size=21171) 2024-11-10T04:25:42,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/bdd063820d9b4737993c3645cb9c0e0f 2024-11-10T04:25:42,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/bdd063820d9b4737993c3645cb9c0e0f as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/bdd063820d9b4737993c3645cb9c0e0f 2024-11-10T04:25:42,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/bdd063820d9b4737993c3645cb9c0e0f, entries=15, sequenceid=279, filesize=20.7 K 2024-11-10T04:25:42,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=10.51 KB/10760 for acfe3a64f0010d1915c5e283bf349529 in 20ms, sequenceid=279, compaction requested=true 2024-11-10T04:25:42,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:42,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acfe3a64f0010d1915c5e283bf349529:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:42,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:42,220 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:42,221 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 181023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:42,221 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): acfe3a64f0010d1915c5e283bf349529/info is initiating minor compaction (all files) 2024-11-10T04:25:42,221 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acfe3a64f0010d1915c5e283bf349529/info in TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:42,221 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/4422e526273a4d10823d7a4ec484e55d, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0af053fbd84440c492cfedfe1a7563fa, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/bdd063820d9b4737993c3645cb9c0e0f] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp, totalSize=176.8 K 2024-11-10T04:25:42,222 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4422e526273a4d10823d7a4ec484e55d, keycount=131, bloomtype=ROW, size=143.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731212721836 2024-11-10T04:25:42,222 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af053fbd84440c492cfedfe1a7563fa, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731212742167 2024-11-10T04:25:42,222 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdd063820d9b4737993c3645cb9c0e0f, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731212742179 2024-11-10T04:25:42,234 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acfe3a64f0010d1915c5e283bf349529#info#compaction#85 average throughput is 39.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:42,234 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0744e64d0e4f426cb9c2406e3e312765 is 1080, key is row0062/info:/1731212721836/Put/seqid=0 2024-11-10T04:25:42,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741870_1046 (size=171177) 2024-11-10T04:25:42,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741870_1046 (size=171177) 2024-11-10T04:25:42,241 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/0744e64d0e4f426cb9c2406e3e312765 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0744e64d0e4f426cb9c2406e3e312765 2024-11-10T04:25:42,246 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acfe3a64f0010d1915c5e283bf349529/info of acfe3a64f0010d1915c5e283bf349529 into 0744e64d0e4f426cb9c2406e3e312765(size=167.2 K), total size for store is 167.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:42,246 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:42,246 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., storeName=acfe3a64f0010d1915c5e283bf349529/info, priority=13, startTime=1731212742220; duration=0sec 2024-11-10T04:25:42,247 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:42,247 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acfe3a64f0010d1915c5e283bf349529:info 2024-11-10T04:25:42,269 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:42,300 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:43,270 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:43,300 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:44,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-10T04:25:44,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/d916e67664334a2b96291cdc365b01da is 1080, key is row0215/info:/1731212742201/Put/seqid=0 2024-11-10T04:25:44,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741871_1047 (size=16839) 2024-11-10T04:25:44,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741871_1047 (size=16839) 2024-11-10T04:25:44,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/d916e67664334a2b96291cdc365b01da 2024-11-10T04:25:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/d916e67664334a2b96291cdc365b01da as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/d916e67664334a2b96291cdc365b01da 2024-11-10T04:25:44,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/d916e67664334a2b96291cdc365b01da, entries=11, sequenceid=294, filesize=16.4 K 2024-11-10T04:25:44,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for acfe3a64f0010d1915c5e283bf349529 in 20ms, sequenceid=294, compaction requested=false 2024-11-10T04:25:44,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:44,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:44,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/760561dfa3c54b97b10fddc169aa1024 is 1080, key is row0226/info:/1731212744219/Put/seqid=0 2024-11-10T04:25:44,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741872_1048 (size=20092) 2024-11-10T04:25:44,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741872_1048 (size=20092) 2024-11-10T04:25:44,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/760561dfa3c54b97b10fddc169aa1024 2024-11-10T04:25:44,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/760561dfa3c54b97b10fddc169aa1024 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/760561dfa3c54b97b10fddc169aa1024 2024-11-10T04:25:44,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/760561dfa3c54b97b10fddc169aa1024, entries=14, sequenceid=311, filesize=19.6 K 2024-11-10T04:25:44,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for acfe3a64f0010d1915c5e283bf349529 in 19ms, sequenceid=311, compaction requested=true 2024-11-10T04:25:44,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:44,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store acfe3a64f0010d1915c5e283bf349529:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:25:44,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:44,260 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:25:44,261 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 208108 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:25:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38277 {}] regionserver.HRegion(8855): Flush requested on acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:44,261 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1541): acfe3a64f0010d1915c5e283bf349529/info is initiating minor compaction (all files) 2024-11-10T04:25:44,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T04:25:44,261 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of acfe3a64f0010d1915c5e283bf349529/info in TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:44,261 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0744e64d0e4f426cb9c2406e3e312765, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/d916e67664334a2b96291cdc365b01da, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/760561dfa3c54b97b10fddc169aa1024] into tmpdir=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp, totalSize=203.2 K 2024-11-10T04:25:44,262 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0744e64d0e4f426cb9c2406e3e312765, keycount=153, bloomtype=ROW, size=167.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731212721836 2024-11-10T04:25:44,262 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting d916e67664334a2b96291cdc365b01da, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731212742201 2024-11-10T04:25:44,262 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] compactions.Compactor(225): Compacting 760561dfa3c54b97b10fddc169aa1024, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731212744219 2024-11-10T04:25:44,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/1cc97743702740a59e718c85effde2ed is 1080, key is row0240/info:/1731212744241/Put/seqid=0 2024-11-10T04:25:44,270 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:44,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741873_1049 (size=20092) 2024-11-10T04:25:44,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741873_1049 (size=20092) 2024-11-10T04:25:44,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/1cc97743702740a59e718c85effde2ed 2024-11-10T04:25:44,279 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): acfe3a64f0010d1915c5e283bf349529#info#compaction#89 average throughput is 60.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:25:44,280 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/3524cdb786864cd0a035beb52fcefa57 is 1080, key is row0062/info:/1731212721836/Put/seqid=0 2024-11-10T04:25:44,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741874_1050 (size=198258) 2024-11-10T04:25:44,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741874_1050 (size=198258) 2024-11-10T04:25:44,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/1cc97743702740a59e718c85effde2ed as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/1cc97743702740a59e718c85effde2ed 2024-11-10T04:25:44,287 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/3524cdb786864cd0a035beb52fcefa57 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3524cdb786864cd0a035beb52fcefa57 2024-11-10T04:25:44,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/1cc97743702740a59e718c85effde2ed, entries=14, sequenceid=328, filesize=19.6 K 2024-11-10T04:25:44,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=3.15 KB/3228 for acfe3a64f0010d1915c5e283bf349529 in 27ms, sequenceid=328, compaction requested=false 2024-11-10T04:25:44,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:44,292 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in acfe3a64f0010d1915c5e283bf349529/info of acfe3a64f0010d1915c5e283bf349529 into 3524cdb786864cd0a035beb52fcefa57(size=193.6 K), total size for store is 213.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:25:44,292 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:44,293 INFO [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., storeName=acfe3a64f0010d1915c5e283bf349529/info, priority=13, startTime=1731212744260; duration=0sec 2024-11-10T04:25:44,293 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:25:44,293 DEBUG [RS:0;d1d9e3766cb1:38277-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: acfe3a64f0010d1915c5e283bf349529:info 2024-11-10T04:25:44,301 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:44,559 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340 2024-11-10T04:25:45,271 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:45,301 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:46,266 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-10T04:25:46,267 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38277%2C1731212698789.1731212746266 2024-11-10T04:25:46,271 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:46,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,273 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212699166 with entries=312, filesize=308.51 KB; new WAL /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212746266 2024-11-10T04:25:46,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741833_1009 (size=315921) 2024-11-10T04:25:46,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741833_1009 (size=315921) 2024-11-10T04:25:46,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44489:44489),(127.0.0.1/127.0.0.1:37577:37577)] 2024-11-10T04:25:46,284 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 265dc97e0d16e22599da5e120f5b06c6: 2024-11-10T04:25:46,284 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acfe3a64f0010d1915c5e283bf349529 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-10T04:25:46,289 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/b1ed5773253b4967bbc22c7325f26d80 is 1080, key is row0254/info:/1731212744262/Put/seqid=0 2024-11-10T04:25:46,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741876_1052 (size=8199) 2024-11-10T04:25:46,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741876_1052 (size=8199) 2024-11-10T04:25:46,294 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/b1ed5773253b4967bbc22c7325f26d80 2024-11-10T04:25:46,298 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/.tmp/info/b1ed5773253b4967bbc22c7325f26d80 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/b1ed5773253b4967bbc22c7325f26d80 2024-11-10T04:25:46,301 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:46,302 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/b1ed5773253b4967bbc22c7325f26d80, entries=3, sequenceid=335, filesize=8.0 K 2024-11-10T04:25:46,303 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for acfe3a64f0010d1915c5e283bf349529 in 19ms, sequenceid=335, compaction requested=true 2024-11-10T04:25:46,304 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for acfe3a64f0010d1915c5e283bf349529: 2024-11-10T04:25:46,304 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB 2024-11-10T04:25:46,307 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/info/92c2e1a6267b4389bb1c2efdde83d66a is 193, key is TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529./info:regioninfo/1731212724616/Put/seqid=0 2024-11-10T04:25:46,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741877_1053 (size=6223) 2024-11-10T04:25:46,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741877_1053 (size=6223) 2024-11-10T04:25:46,312 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/info/92c2e1a6267b4389bb1c2efdde83d66a 2024-11-10T04:25:46,316 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/.tmp/info/92c2e1a6267b4389bb1c2efdde83d66a as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/info/92c2e1a6267b4389bb1c2efdde83d66a 2024-11-10T04:25:46,320 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/info/92c2e1a6267b4389bb1c2efdde83d66a, entries=5, sequenceid=21, filesize=6.1 K 2024-11-10T04:25:46,321 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 17ms, sequenceid=21, compaction requested=false 2024-11-10T04:25:46,321 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T04:25:46,321 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C38277%2C1731212698789.1731212746321 2024-11-10T04:25:46,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,326 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212746266 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212746321 2024-11-10T04:25:46,326 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44489:44489),(127.0.0.1/127.0.0.1:37577:37577)] 2024-11-10T04:25:46,327 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212746266 is not closed yet, will try archiving it next time 2024-11-10T04:25:46,327 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212699166 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/oldWALs/d1d9e3766cb1%2C38277%2C1731212698789.1731212699166 2024-11-10T04:25:46,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741875_1051 (size=731) 2024-11-10T04:25:46,327 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T04:25:46,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741875_1051 (size=731) 2024-11-10T04:25:46,328 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/WALs/d1d9e3766cb1,38277,1731212698789/d1d9e3766cb1%2C38277%2C1731212698789.1731212746266 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/oldWALs/d1d9e3766cb1%2C38277%2C1731212698789.1731212746266 2024-11-10T04:25:46,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:25:46,428 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:25:46,428 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:25:46,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:46,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:46,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:25:46,428 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:25:46,428 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1535039275, stopped=false 2024-11-10T04:25:46,429 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,40483,1731212698744 2024-11-10T04:25:46,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:25:46,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:25:46,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:46,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:46,431 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:25:46,431 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:25:46,431 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:25:46,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:46,431 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,38277,1731212698789' ***** 2024-11-10T04:25:46,431 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:25:46,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:25:46,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:25:46,432 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:25:46,432 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:25:46,432 INFO [RS:0;d1d9e3766cb1:38277 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:25:46,432 INFO [RS:0;d1d9e3766cb1:38277 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:25:46,432 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(3091): Received CLOSE for 265dc97e0d16e22599da5e120f5b06c6 2024-11-10T04:25:46,432 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(3091): Received CLOSE for acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:46,432 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:46,432 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:25:46,433 INFO [RS:0;d1d9e3766cb1:38277 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:38277. 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 265dc97e0d16e22599da5e120f5b06c6, disabling compactions & flushes 2024-11-10T04:25:46,433 DEBUG [RS:0;d1d9e3766cb1:38277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:25:46,433 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:46,433 DEBUG [RS:0;d1d9e3766cb1:38277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. after waiting 0 ms 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:46,433 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:25:46,433 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:25:46,433 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:25:46,433 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:25:46,433 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-10T04:25:46,433 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1325): Online Regions={265dc97e0d16e22599da5e120f5b06c6=TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6., acfe3a64f0010d1915c5e283bf349529=TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T04:25:46,433 DEBUG [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 265dc97e0d16e22599da5e120f5b06c6, acfe3a64f0010d1915c5e283bf349529 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:25:46,433 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:25:46,433 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:25:46,433 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031->hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409-bottom] to archive 2024-11-10T04:25:46,434 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:25:46,436 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:46,436 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d1d9e3766cb1:40483 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T04:25:46,437 WARN [StoreCloser-TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-10T04:25:46,437 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-10T04:25:46,438 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:25:46,438 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:25:46,438 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212746433Running coprocessor pre-close hooks at 1731212746433Disabling compacts and flushes for region at 1731212746433Disabling writes for close at 1731212746433Writing region close event to WAL at 1731212746434 (+1 ms)Running coprocessor post-close hooks at 1731212746438 (+4 ms)Closed at 1731212746438 2024-11-10T04:25:46,438 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:25:46,440 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/265dc97e0d16e22599da5e120f5b06c6/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=126 2024-11-10T04:25:46,440 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:46,441 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 265dc97e0d16e22599da5e120f5b06c6: Waiting for close lock at 1731212746432Running coprocessor pre-close hooks at 1731212746432Disabling compacts and flushes for region at 1731212746432Disabling writes for close at 1731212746433 (+1 ms)Writing region close event to WAL at 1731212746437 (+4 ms)Running coprocessor post-close hooks at 1731212746440 (+3 ms)Closed at 1731212746440 2024-11-10T04:25:46,441 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731212723909.265dc97e0d16e22599da5e120f5b06c6. 2024-11-10T04:25:46,441 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing acfe3a64f0010d1915c5e283bf349529, disabling compactions & flushes 2024-11-10T04:25:46,441 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:46,441 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:46,441 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. after waiting 0 ms 2024-11-10T04:25:46,441 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:46,441 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031->hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/4c285b8b061af9a7de1e4f817de4f031/info/3890706b877b49fa88a9131260dae409-top, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/c044c89d18484b57a1a4b9a853478321, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/25d20b61a5244f4bb02956413e67e4bd, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/62f87626d6e7427d91c7990fb3620bb2, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5c09b10b45704e8c87b58eb6989416f7, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/efb8fbf85c094c5c8227ec8dc4b72673, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3d7a34e1b7d54e87a77fe3bff08b2edf, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/a96b162f51db4c77b501faeacc2a2296, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5ab57b3d4ad94aacaddd064654082a36, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/8a08062c0d3a4beba7063ce5c9e2c80f, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0c58865d234f488a9b0ac543640046d0, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/047ddf93e0a34b2bbb72e9c883149968, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/4422e526273a4d10823d7a4ec484e55d, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/77770f75b92e4c56aed2d7f0d5e8124c, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0af053fbd84440c492cfedfe1a7563fa, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0744e64d0e4f426cb9c2406e3e312765, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/bdd063820d9b4737993c3645cb9c0e0f, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/d916e67664334a2b96291cdc365b01da, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/760561dfa3c54b97b10fddc169aa1024] to archive 2024-11-10T04:25:46,442 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:25:46,444 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3890706b877b49fa88a9131260dae409.4c285b8b061af9a7de1e4f817de4f031 2024-11-10T04:25:46,445 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-33a13a8dc1064a8395571b696265c5bf 2024-11-10T04:25:46,446 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-2911a4caf4dc4114ad274d77990e2273 2024-11-10T04:25:46,447 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/c044c89d18484b57a1a4b9a853478321 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/c044c89d18484b57a1a4b9a853478321 2024-11-10T04:25:46,448 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/TestLogRolling-testLogRolling=4c285b8b061af9a7de1e4f817de4f031-34e0bb4e8cf543d0ad146e3800c8f62a 2024-11-10T04:25:46,450 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/25d20b61a5244f4bb02956413e67e4bd to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/25d20b61a5244f4bb02956413e67e4bd 2024-11-10T04:25:46,451 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/62f87626d6e7427d91c7990fb3620bb2 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/62f87626d6e7427d91c7990fb3620bb2 2024-11-10T04:25:46,452 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5c09b10b45704e8c87b58eb6989416f7 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5c09b10b45704e8c87b58eb6989416f7 2024-11-10T04:25:46,453 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/efb8fbf85c094c5c8227ec8dc4b72673 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/efb8fbf85c094c5c8227ec8dc4b72673 2024-11-10T04:25:46,454 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3d7a34e1b7d54e87a77fe3bff08b2edf to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/3d7a34e1b7d54e87a77fe3bff08b2edf 2024-11-10T04:25:46,454 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/a96b162f51db4c77b501faeacc2a2296 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/a96b162f51db4c77b501faeacc2a2296 2024-11-10T04:25:46,456 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5ab57b3d4ad94aacaddd064654082a36 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/5ab57b3d4ad94aacaddd064654082a36 2024-11-10T04:25:46,457 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/8a08062c0d3a4beba7063ce5c9e2c80f to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/8a08062c0d3a4beba7063ce5c9e2c80f 2024-11-10T04:25:46,458 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0c58865d234f488a9b0ac543640046d0 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0c58865d234f488a9b0ac543640046d0 2024-11-10T04:25:46,459 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/047ddf93e0a34b2bbb72e9c883149968 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/047ddf93e0a34b2bbb72e9c883149968 2024-11-10T04:25:46,460 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/4422e526273a4d10823d7a4ec484e55d to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/4422e526273a4d10823d7a4ec484e55d 2024-11-10T04:25:46,461 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/77770f75b92e4c56aed2d7f0d5e8124c to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/77770f75b92e4c56aed2d7f0d5e8124c 2024-11-10T04:25:46,462 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0af053fbd84440c492cfedfe1a7563fa to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0af053fbd84440c492cfedfe1a7563fa 2024-11-10T04:25:46,463 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0744e64d0e4f426cb9c2406e3e312765 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/0744e64d0e4f426cb9c2406e3e312765 2024-11-10T04:25:46,464 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/bdd063820d9b4737993c3645cb9c0e0f to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/bdd063820d9b4737993c3645cb9c0e0f 2024-11-10T04:25:46,465 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/d916e67664334a2b96291cdc365b01da to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/d916e67664334a2b96291cdc365b01da 2024-11-10T04:25:46,466 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/760561dfa3c54b97b10fddc169aa1024 to hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/archive/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/info/760561dfa3c54b97b10fddc169aa1024 2024-11-10T04:25:46,466 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c044c89d18484b57a1a4b9a853478321=43081, 25d20b61a5244f4bb02956413e67e4bd=12516, 62f87626d6e7427d91c7990fb3620bb2=65889, 5c09b10b45704e8c87b58eb6989416f7=20078, efb8fbf85c094c5c8227ec8dc4b72673=17906, 3d7a34e1b7d54e87a77fe3bff08b2edf=94096, a96b162f51db4c77b501faeacc2a2296=20078, 5ab57b3d4ad94aacaddd064654082a36=20078, 8a08062c0d3a4beba7063ce5c9e2c80f=116840, 0c58865d234f488a9b0ac543640046d0=12516, 047ddf93e0a34b2bbb72e9c883149968=21156, 4422e526273a4d10823d7a4ec484e55d=147331, 77770f75b92e4c56aed2d7f0d5e8124c=19000, 0af053fbd84440c492cfedfe1a7563fa=12521, 0744e64d0e4f426cb9c2406e3e312765=171177, bdd063820d9b4737993c3645cb9c0e0f=21171, d916e67664334a2b96291cdc365b01da=16839, 760561dfa3c54b97b10fddc169aa1024=20092] 2024-11-10T04:25:46,470 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/data/default/TestLogRolling-testLogRolling/acfe3a64f0010d1915c5e283bf349529/recovered.edits/338.seqid, newMaxSeqId=338, maxSeqId=126 2024-11-10T04:25:46,470 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:46,470 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for acfe3a64f0010d1915c5e283bf349529: Waiting for close lock at 1731212746441Running coprocessor pre-close hooks at 1731212746441Disabling compacts and flushes for region at 1731212746441Disabling writes for close at 1731212746441Writing region close event to WAL at 1731212746467 (+26 ms)Running coprocessor post-close hooks at 1731212746470 (+3 ms)Closed at 1731212746470 2024-11-10T04:25:46,470 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731212723909.acfe3a64f0010d1915c5e283bf349529. 2024-11-10T04:25:46,633 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,38277,1731212698789; all regions closed. 2024-11-10T04:25:46,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741834_1010 (size=8107) 2024-11-10T04:25:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741834_1010 (size=8107) 2024-11-10T04:25:46,638 DEBUG [RS:0;d1d9e3766cb1:38277 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/oldWALs 2024-11-10T04:25:46,638 INFO [RS:0;d1d9e3766cb1:38277 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38277%2C1731212698789.meta:.meta(num 1731212699545) 2024-11-10T04:25:46,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741878_1054 (size=780) 2024-11-10T04:25:46,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741878_1054 (size=780) 2024-11-10T04:25:46,643 DEBUG [RS:0;d1d9e3766cb1:38277 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/oldWALs 2024-11-10T04:25:46,643 INFO [RS:0;d1d9e3766cb1:38277 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C38277%2C1731212698789:(num 1731212746321) 2024-11-10T04:25:46,643 DEBUG [RS:0;d1d9e3766cb1:38277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:46,643 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:25:46,643 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:25:46,643 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:25:46,643 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:25:46,643 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:25:46,643 INFO [RS:0;d1d9e3766cb1:38277 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38277 2024-11-10T04:25:46,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,38277,1731212698789 2024-11-10T04:25:46,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:25:46,646 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:25:46,647 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,38277,1731212698789] 2024-11-10T04:25:46,649 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,38277,1731212698789 already deleted, retry=false 2024-11-10T04:25:46,649 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,38277,1731212698789 expired; onlineServers=0 2024-11-10T04:25:46,649 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,40483,1731212698744' ***** 2024-11-10T04:25:46,649 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:25:46,649 INFO [M:0;d1d9e3766cb1:40483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:25:46,649 INFO [M:0;d1d9e3766cb1:40483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:25:46,650 DEBUG [M:0;d1d9e3766cb1:40483 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:25:46,650 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:25:46,650 DEBUG [M:0;d1d9e3766cb1:40483 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:25:46,650 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212698942 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212698942,5,FailOnTimeoutGroup] 2024-11-10T04:25:46,650 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212698942 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212698942,5,FailOnTimeoutGroup] 2024-11-10T04:25:46,650 INFO [M:0;d1d9e3766cb1:40483 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:25:46,650 INFO [M:0;d1d9e3766cb1:40483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:25:46,650 DEBUG [M:0;d1d9e3766cb1:40483 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:25:46,650 INFO [M:0;d1d9e3766cb1:40483 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:25:46,650 INFO [M:0;d1d9e3766cb1:40483 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:25:46,651 INFO [M:0;d1d9e3766cb1:40483 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:25:46,651 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:25:46,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:25:46,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:46,651 DEBUG [M:0;d1d9e3766cb1:40483 {}] zookeeper.ZKUtil(347): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:25:46,651 WARN [M:0;d1d9e3766cb1:40483 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:25:46,652 INFO [M:0;d1d9e3766cb1:40483 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/.lastflushedseqids 2024-11-10T04:25:46,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741879_1055 (size=228) 2024-11-10T04:25:46,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741879_1055 (size=228) 2024-11-10T04:25:46,657 INFO [M:0;d1d9e3766cb1:40483 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:25:46,657 INFO [M:0;d1d9e3766cb1:40483 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:25:46,657 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:25:46,657 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:46,657 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:46,657 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:25:46,657 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:46,657 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=53.71 KB heapSize=65.93 KB 2024-11-10T04:25:46,673 DEBUG [M:0;d1d9e3766cb1:40483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e90f02e9698f4c83a38bb864fb7252b5 is 82, key is hbase:meta,,1/info:regioninfo/1731212699574/Put/seqid=0 2024-11-10T04:25:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741880_1056 (size=5672) 2024-11-10T04:25:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741880_1056 (size=5672) 2024-11-10T04:25:46,678 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e90f02e9698f4c83a38bb864fb7252b5 2024-11-10T04:25:46,696 DEBUG [M:0;d1d9e3766cb1:40483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e58d818a377f4d3cba9b4ae6433abc88 is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731212699989/Put/seqid=0 2024-11-10T04:25:46,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741881_1057 (size=7681) 2024-11-10T04:25:46,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741881_1057 (size=7681) 2024-11-10T04:25:46,701 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.11 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e58d818a377f4d3cba9b4ae6433abc88 2024-11-10T04:25:46,705 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e58d818a377f4d3cba9b4ae6433abc88 2024-11-10T04:25:46,719 DEBUG [M:0;d1d9e3766cb1:40483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97ad285af4fd48599868d80dcf426392 is 69, key is d1d9e3766cb1,38277,1731212698789/rs:state/1731212699023/Put/seqid=0 2024-11-10T04:25:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741882_1058 (size=5156) 2024-11-10T04:25:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741882_1058 (size=5156) 2024-11-10T04:25:46,724 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97ad285af4fd48599868d80dcf426392 2024-11-10T04:25:46,742 DEBUG [M:0;d1d9e3766cb1:40483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a3b4f38bb10a4abaa9eaaeb42d1bc92c is 52, key is load_balancer_on/state:d/1731212699612/Put/seqid=0 2024-11-10T04:25:46,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741883_1059 (size=5056) 2024-11-10T04:25:46,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741883_1059 (size=5056) 2024-11-10T04:25:46,747 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a3b4f38bb10a4abaa9eaaeb42d1bc92c 2024-11-10T04:25:46,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:46,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38277-0x101906cd9d20001, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:46,747 INFO [RS:0;d1d9e3766cb1:38277 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:25:46,748 INFO [RS:0;d1d9e3766cb1:38277 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,38277,1731212698789; zookeeper connection closed. 2024-11-10T04:25:46,748 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4217d878 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4217d878 2024-11-10T04:25:46,748 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:25:46,751 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e90f02e9698f4c83a38bb864fb7252b5 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e90f02e9698f4c83a38bb864fb7252b5 2024-11-10T04:25:46,755 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e90f02e9698f4c83a38bb864fb7252b5, entries=8, sequenceid=129, filesize=5.5 K 2024-11-10T04:25:46,756 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e58d818a377f4d3cba9b4ae6433abc88 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e58d818a377f4d3cba9b4ae6433abc88 2024-11-10T04:25:46,760 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e58d818a377f4d3cba9b4ae6433abc88 2024-11-10T04:25:46,760 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e58d818a377f4d3cba9b4ae6433abc88, entries=14, sequenceid=129, filesize=7.5 K 2024-11-10T04:25:46,761 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97ad285af4fd48599868d80dcf426392 as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97ad285af4fd48599868d80dcf426392 2024-11-10T04:25:46,765 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97ad285af4fd48599868d80dcf426392, entries=1, sequenceid=129, filesize=5.0 K 2024-11-10T04:25:46,766 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a3b4f38bb10a4abaa9eaaeb42d1bc92c as hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a3b4f38bb10a4abaa9eaaeb42d1bc92c 2024-11-10T04:25:46,769 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34945/user/jenkins/test-data/f79d29ff-343b-dabc-f0c9-416aa95d19a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a3b4f38bb10a4abaa9eaaeb42d1bc92c, entries=1, sequenceid=129, filesize=4.9 K 2024-11-10T04:25:46,770 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=129, compaction requested=false 2024-11-10T04:25:46,772 INFO [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:46,772 DEBUG [M:0;d1d9e3766cb1:40483 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212746657Disabling compacts and flushes for region at 1731212746657Disabling writes for close at 1731212746657Obtaining lock to block concurrent updates at 1731212746657Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212746657Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=54997, getHeapSize=67448, getOffHeapSize=0, getCellsCount=152 at 1731212746658 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212746658Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212746659 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212746673 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212746673Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212746682 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212746696 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212746696Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212746706 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212746719 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212746719Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731212746728 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731212746742 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731212746742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b39fa1: reopening flushed file at 1731212746751 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a46fecf: reopening flushed file at 1731212746755 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23d03fc3: reopening flushed file at 1731212746760 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1745aec0: reopening flushed file at 1731212746765 (+5 ms)Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=129, compaction requested=false at 1731212746770 (+5 ms)Writing region close event to WAL at 1731212746772 (+2 ms)Closed at 1731212746772 2024-11-10T04:25:46,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44267 is added to blk_1073741830_1006 (size=63927) 2024-11-10T04:25:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741830_1006 (size=63927) 2024-11-10T04:25:46,775 INFO [M:0;d1d9e3766cb1:40483 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:25:46,775 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:25:46,775 INFO [M:0;d1d9e3766cb1:40483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40483 2024-11-10T04:25:46,775 INFO [M:0;d1d9e3766cb1:40483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:25:46,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:46,877 INFO [M:0;d1d9e3766cb1:40483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:25:46,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40483-0x101906cd9d20000, quorum=127.0.0.1:49477, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:46,879 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18d1ee92{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:25:46,880 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78512cf7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:25:46,880 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:25:46,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41a74ab6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:25:46,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64d2170c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir/,STOPPED} 2024-11-10T04:25:46,882 WARN [BP-1653749568-172.17.0.2-1731212698081 heartbeating to localhost/127.0.0.1:34945 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:25:46,882 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:25:46,882 WARN [BP-1653749568-172.17.0.2-1731212698081 heartbeating to localhost/127.0.0.1:34945 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1653749568-172.17.0.2-1731212698081 (Datanode Uuid 8b4f7f24-ba32-4fcc-86cd-1a57d75c6712) service to localhost/127.0.0.1:34945 2024-11-10T04:25:46,882 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:25:46,882 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data3/current/BP-1653749568-172.17.0.2-1731212698081 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:46,883 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data4/current/BP-1653749568-172.17.0.2-1731212698081 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:46,883 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:25:46,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@444d0b71{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:25:46,885 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3bc081d8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:25:46,885 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:25:46,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ecf816b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:25:46,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@374d3611{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir/,STOPPED} 2024-11-10T04:25:46,888 WARN [BP-1653749568-172.17.0.2-1731212698081 heartbeating to localhost/127.0.0.1:34945 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:25:46,888 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:25:46,888 WARN [BP-1653749568-172.17.0.2-1731212698081 heartbeating to localhost/127.0.0.1:34945 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1653749568-172.17.0.2-1731212698081 (Datanode Uuid e0caa0eb-56b4-4a9a-b9a6-08ef6c379ac1) service to localhost/127.0.0.1:34945 2024-11-10T04:25:46,888 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:25:46,888 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data1/current/BP-1653749568-172.17.0.2-1731212698081 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:46,888 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/cluster_e1fed399-835d-0e65-b40b-c8cfe99fbad0/data/data2/current/BP-1653749568-172.17.0.2-1731212698081 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:46,889 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:25:46,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@319ddf98{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:25:46,895 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@371dfe16{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:25:46,895 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:25:46,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a0fc9ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:25:46,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35e03861{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir/,STOPPED} 2024-11-10T04:25:46,903 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:25:46,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:25:46,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 206) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34945 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34945 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34945 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:34945 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34945 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34945 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34945 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34945 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=512 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=41 (was 55), ProcessCount=11 (was 11), AvailableMemoryMB=6356 (was 6393) 2024-11-10T04:25:46,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=512, MaxFileDescriptor=1048576, SystemLoadAverage=41, ProcessCount=11, AvailableMemoryMB=6357 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.log.dir so I do NOT create it in target/test-data/59db72f9-1a2b-c827-f276-588a01332747 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ed1bb123-4829-29b1-f17f-791132f93b9e/hadoop.tmp.dir so I do NOT create it in target/test-data/59db72f9-1a2b-c827-f276-588a01332747 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417, deleteOnExit=true 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/test.cache.data in system properties and HBase conf 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:25:46,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:25:46,946 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:25:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:25:46,960 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:25:47,015 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:25:47,019 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:25:47,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:25:47,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:25:47,020 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:25:47,020 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:25:47,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5031c9de{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:25:47,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1cb0de45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:25:47,036 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:25:47,134 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b418861{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/java.io.tmpdir/jetty-localhost-33511-hadoop-hdfs-3_4_1-tests_jar-_-any-11673742054939984113/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:25:47,135 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7cdb81da{HTTP/1.1, (http/1.1)}{localhost:33511} 2024-11-10T04:25:47,135 INFO [Time-limited test {}] server.Server(415): Started @286680ms 2024-11-10T04:25:47,147 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T04:25:47,197 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:25:47,199 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:25:47,200 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:25:47,200 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:25:47,200 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:25:47,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6162294b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:25:47,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41f14207{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:25:47,272 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:47,302 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:47,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@61266769{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/java.io.tmpdir/jetty-localhost-45161-hadoop-hdfs-3_4_1-tests_jar-_-any-2959140036559095589/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:25:47,314 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49428dc3{HTTP/1.1, (http/1.1)}{localhost:45161} 2024-11-10T04:25:47,314 INFO [Time-limited test {}] server.Server(415): Started @286859ms 2024-11-10T04:25:47,315 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:25:47,344 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:25:47,346 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:25:47,347 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:25:47,347 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:25:47,347 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:25:47,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47b9368{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:25:47,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32c86c86{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:25:47,404 WARN [Thread-2488 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data1/current/BP-1332666732-172.17.0.2-1731212746965/current, will proceed with Du for space computation calculation, 2024-11-10T04:25:47,404 WARN [Thread-2489 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data2/current/BP-1332666732-172.17.0.2-1731212746965/current, will proceed with Du for space computation calculation, 2024-11-10T04:25:47,426 WARN [Thread-2467 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:25:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa54c491a88d87836 with lease ID 0xdea0f90072009840: Processing first storage report for DS-625eac4c-27d1-45eb-a8af-4ad38af9dcbe from datanode DatanodeRegistration(127.0.0.1:43087, datanodeUuid=d78f89ed-9edf-4e79-b4d0-d944a642d45a, infoPort=44665, infoSecurePort=0, ipcPort=36363, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965) 2024-11-10T04:25:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa54c491a88d87836 with lease ID 0xdea0f90072009840: from storage DS-625eac4c-27d1-45eb-a8af-4ad38af9dcbe node DatanodeRegistration(127.0.0.1:43087, datanodeUuid=d78f89ed-9edf-4e79-b4d0-d944a642d45a, infoPort=44665, infoSecurePort=0, ipcPort=36363, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:25:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa54c491a88d87836 with lease ID 0xdea0f90072009840: Processing first storage report for DS-fc6b856c-6b91-49bd-9bde-39af4b08d596 from datanode DatanodeRegistration(127.0.0.1:43087, datanodeUuid=d78f89ed-9edf-4e79-b4d0-d944a642d45a, infoPort=44665, infoSecurePort=0, ipcPort=36363, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965) 2024-11-10T04:25:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa54c491a88d87836 with lease ID 0xdea0f90072009840: from storage DS-fc6b856c-6b91-49bd-9bde-39af4b08d596 node DatanodeRegistration(127.0.0.1:43087, datanodeUuid=d78f89ed-9edf-4e79-b4d0-d944a642d45a, infoPort=44665, infoSecurePort=0, ipcPort=36363, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:25:47,469 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4cb9612e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/java.io.tmpdir/jetty-localhost-40279-hadoop-hdfs-3_4_1-tests_jar-_-any-14896112634894649135/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:25:47,469 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@186f146f{HTTP/1.1, (http/1.1)}{localhost:40279} 2024-11-10T04:25:47,469 INFO [Time-limited test {}] server.Server(415): Started @287014ms 2024-11-10T04:25:47,470 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:25:47,559 WARN [Thread-2514 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data3/current/BP-1332666732-172.17.0.2-1731212746965/current, will proceed with Du for space computation calculation, 2024-11-10T04:25:47,559 WARN [Thread-2515 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data4/current/BP-1332666732-172.17.0.2-1731212746965/current, will proceed with Du for space computation calculation, 2024-11-10T04:25:47,575 WARN [Thread-2503 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:25:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x749c01a731ec38de with lease ID 0xdea0f90072009841: Processing first storage report for DS-aae12712-f91a-40b1-b87e-db9baf83514b from datanode DatanodeRegistration(127.0.0.1:40157, datanodeUuid=73d2af8d-243a-4b02-90dc-952d0380b620, infoPort=39207, infoSecurePort=0, ipcPort=34601, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965) 2024-11-10T04:25:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x749c01a731ec38de with lease ID 0xdea0f90072009841: from storage DS-aae12712-f91a-40b1-b87e-db9baf83514b node DatanodeRegistration(127.0.0.1:40157, datanodeUuid=73d2af8d-243a-4b02-90dc-952d0380b620, infoPort=39207, infoSecurePort=0, ipcPort=34601, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:25:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x749c01a731ec38de with lease ID 0xdea0f90072009841: Processing first storage report for DS-d5c1d826-0bfb-416e-b800-e1baba334d13 from datanode DatanodeRegistration(127.0.0.1:40157, datanodeUuid=73d2af8d-243a-4b02-90dc-952d0380b620, infoPort=39207, infoSecurePort=0, ipcPort=34601, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965) 2024-11-10T04:25:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x749c01a731ec38de with lease ID 0xdea0f90072009841: from storage DS-d5c1d826-0bfb-416e-b800-e1baba334d13 node DatanodeRegistration(127.0.0.1:40157, datanodeUuid=73d2af8d-243a-4b02-90dc-952d0380b620, infoPort=39207, infoSecurePort=0, ipcPort=34601, storageInfo=lv=-57;cid=testClusterID;nsid=2038864379;c=1731212746965), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:25:47,591 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747 2024-11-10T04:25:47,594 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/zookeeper_0, clientPort=55780, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:25:47,595 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55780 2024-11-10T04:25:47,595 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:47,596 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:47,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:25:47,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:25:47,605 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4 with version=8 2024-11-10T04:25:47,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45907/user/jenkins/test-data/a21fe913-2ca7-678d-ad7a-a3cd4f7c5dce/hbase-staging 2024-11-10T04:25:47,607 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:25:47,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:25:47,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:25:47,607 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:25:47,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:25:47,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:25:47,607 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:25:47,607 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:25:47,608 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36701 2024-11-10T04:25:47,609 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36701 connecting to ZooKeeper ensemble=127.0.0.1:55780 2024-11-10T04:25:47,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:367010x0, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:25:47,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36701-0x101906d98b20000 connected 2024-11-10T04:25:47,630 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:47,631 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:47,633 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:25:47,633 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4, hbase.cluster.distributed=false 2024-11-10T04:25:47,635 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:25:47,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36701 2024-11-10T04:25:47,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36701 2024-11-10T04:25:47,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36701 2024-11-10T04:25:47,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36701 2024-11-10T04:25:47,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36701 2024-11-10T04:25:47,651 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:25:47,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:25:47,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:25:47,651 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:25:47,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:25:47,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:25:47,651 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:25:47,651 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:25:47,652 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46057 2024-11-10T04:25:47,653 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46057 connecting to ZooKeeper ensemble=127.0.0.1:55780 2024-11-10T04:25:47,653 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:47,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:47,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460570x0, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:25:47,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:25:47,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46057-0x101906d98b20001 connected 2024-11-10T04:25:47,660 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:25:47,660 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:25:47,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:25:47,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:25:47,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46057 2024-11-10T04:25:47,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46057 2024-11-10T04:25:47,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46057 2024-11-10T04:25:47,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46057 2024-11-10T04:25:47,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46057 2024-11-10T04:25:47,674 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:36701 2024-11-10T04:25:47,674 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:47,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:25:47,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:25:47,677 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:47,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:25:47,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,679 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:25:47,679 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,36701,1731212747607 from backup master directory 2024-11-10T04:25:47,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:47,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:25:47,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:25:47,681 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:25:47,681 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:47,684 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/hbase.id] with ID: dcf74f63-ba5e-45e3-87c9-5523cf421b5a 2024-11-10T04:25:47,684 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/.tmp/hbase.id 2024-11-10T04:25:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:25:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:25:47,691 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/.tmp/hbase.id]:[hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/hbase.id] 2024-11-10T04:25:47,700 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:47,700 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:25:47,701 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T04:25:47,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:25:47,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:25:47,709 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:25:47,710 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:25:47,710 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:25:47,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:25:47,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:25:47,718 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store 2024-11-10T04:25:47,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:25:47,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:25:47,726 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:25:47,726 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:25:47,726 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:47,726 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:47,726 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:25:47,726 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:47,726 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:47,726 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212747726Disabling compacts and flushes for region at 1731212747726Disabling writes for close at 1731212747726Writing region close event to WAL at 1731212747726Closed at 1731212747726 2024-11-10T04:25:47,727 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/.initializing 2024-11-10T04:25:47,727 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/WALs/d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:47,729 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C36701%2C1731212747607, suffix=, logDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/WALs/d1d9e3766cb1,36701,1731212747607, archiveDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/oldWALs, maxLogs=10 2024-11-10T04:25:47,729 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C36701%2C1731212747607.1731212747729 2024-11-10T04:25:47,733 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/WALs/d1d9e3766cb1,36701,1731212747607/d1d9e3766cb1%2C36701%2C1731212747607.1731212747729 2024-11-10T04:25:47,736 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39207:39207),(127.0.0.1/127.0.0.1:44665:44665)] 2024-11-10T04:25:47,738 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:25:47,738 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:25:47,738 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,738 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:25:47,741 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:47,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:25:47,742 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:25:47,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:25:47,744 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:25:47,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:25:47,745 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:25:47,745 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,746 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,746 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,747 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,747 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,748 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:25:47,749 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:25:47,751 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:25:47,751 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842525, jitterRate=0.07132625579833984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:25:47,752 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212747739Initializing all the Stores at 1731212747739Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212747739Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212747739Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212747740 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212747740Cleaning up temporary data from old regions at 1731212747747 (+7 ms)Region opened successfully at 1731212747752 (+5 ms) 2024-11-10T04:25:47,752 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:25:47,754 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2314c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:25:47,755 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:25:47,755 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:25:47,755 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:25:47,755 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:25:47,756 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T04:25:47,756 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T04:25:47,756 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:25:47,758 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:25:47,758 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:25:47,760 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:25:47,760 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:25:47,761 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:25:47,762 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:25:47,763 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:25:47,763 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:25:47,765 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:25:47,765 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:25:47,767 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:25:47,768 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:25:47,769 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:25:47,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:25:47,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:25:47,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,772 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,36701,1731212747607, sessionid=0x101906d98b20000, setting cluster-up flag (Was=false) 2024-11-10T04:25:47,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,780 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:25:47,781 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:47,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:47,790 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:25:47,791 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:47,792 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:25:47,794 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:25:47,794 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:25:47,794 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:25:47,794 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,36701,1731212747607 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:25:47,795 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:25:47,795 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:25:47,795 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:25:47,795 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:25:47,796 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:25:47,796 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,796 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:25:47,796 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,796 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212777796 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,797 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:25:47,797 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:25:47,797 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:25:47,798 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:25:47,798 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:25:47,798 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212747798,5,FailOnTimeoutGroup] 2024-11-10T04:25:47,798 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,798 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212747798,5,FailOnTimeoutGroup] 2024-11-10T04:25:47,798 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,798 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:25:47,798 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,798 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,798 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:25:47,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:25:47,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:25:47,804 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:25:47,804 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4 2024-11-10T04:25:47,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:25:47,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741832_1008 (size=32) 2024-11-10T04:25:47,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:25:47,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:25:47,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:25:47,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:47,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:25:47,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:25:47,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:47,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:25:47,814 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:25:47,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:47,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:25:47,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:25:47,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:47,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:47,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:25:47,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740 2024-11-10T04:25:47,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740 2024-11-10T04:25:47,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:25:47,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:25:47,819 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:25:47,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:25:47,821 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:25:47,821 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804466, jitterRate=0.022931665182113647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:25:47,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212747810Initializing all the Stores at 1731212747811 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212747811Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212747811Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212747811Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212747811Cleaning up temporary data from old regions at 1731212747818 (+7 ms)Region opened successfully at 1731212747822 (+4 ms) 2024-11-10T04:25:47,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:25:47,822 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:25:47,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:25:47,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:25:47,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:25:47,822 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:25:47,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212747822Disabling compacts and flushes for region at 1731212747822Disabling writes for close at 1731212747822Writing region close event to WAL at 1731212747822Closed at 1731212747822 2024-11-10T04:25:47,823 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:25:47,823 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:25:47,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:25:47,824 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:25:47,825 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:25:47,864 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(746): ClusterId : dcf74f63-ba5e-45e3-87c9-5523cf421b5a 2024-11-10T04:25:47,864 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:25:47,866 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:25:47,867 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:25:47,869 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:25:47,869 DEBUG [RS:0;d1d9e3766cb1:46057 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cf2c0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:25:47,881 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:46057 2024-11-10T04:25:47,881 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:25:47,881 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:25:47,881 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:25:47,882 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,36701,1731212747607 with port=46057, startcode=1731212747651 2024-11-10T04:25:47,882 DEBUG [RS:0;d1d9e3766cb1:46057 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:25:47,884 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:25:47,884 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36701 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:47,884 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36701 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:47,886 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4 2024-11-10T04:25:47,886 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37439 2024-11-10T04:25:47,886 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:25:47,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:25:47,888 DEBUG [RS:0;d1d9e3766cb1:46057 {}] zookeeper.ZKUtil(111): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:47,888 WARN [RS:0;d1d9e3766cb1:46057 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:25:47,888 INFO [RS:0;d1d9e3766cb1:46057 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:25:47,888 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:47,889 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,46057,1731212747651] 2024-11-10T04:25:47,892 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:25:47,893 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:25:47,893 INFO [RS:0;d1d9e3766cb1:46057 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:25:47,893 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,894 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:25:47,894 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:25:47,894 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,894 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:25:47,895 DEBUG [RS:0;d1d9e3766cb1:46057 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:25:47,895 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,895 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,895 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,895 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,895 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,895 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,46057,1731212747651-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:25:47,909 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:25:47,909 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,46057,1731212747651-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,909 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,909 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.Replication(171): d1d9e3766cb1,46057,1731212747651 started 2024-11-10T04:25:47,923 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:47,923 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,46057,1731212747651, RpcServer on d1d9e3766cb1/172.17.0.2:46057, sessionid=0x101906d98b20001 2024-11-10T04:25:47,923 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:25:47,923 DEBUG [RS:0;d1d9e3766cb1:46057 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:47,923 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,46057,1731212747651' 2024-11-10T04:25:47,923 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:25:47,923 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:25:47,924 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:25:47,924 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:25:47,924 DEBUG [RS:0;d1d9e3766cb1:46057 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:47,924 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,46057,1731212747651' 2024-11-10T04:25:47,924 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:25:47,924 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:25:47,925 DEBUG [RS:0;d1d9e3766cb1:46057 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:25:47,925 INFO [RS:0;d1d9e3766cb1:46057 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:25:47,925 INFO [RS:0;d1d9e3766cb1:46057 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:25:47,976 WARN [d1d9e3766cb1:36701 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:25:48,026 INFO [RS:0;d1d9e3766cb1:46057 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C46057%2C1731212747651, suffix=, logDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/d1d9e3766cb1,46057,1731212747651, archiveDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/oldWALs, maxLogs=32 2024-11-10T04:25:48,027 INFO [RS:0;d1d9e3766cb1:46057 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C46057%2C1731212747651.1731212748027 2024-11-10T04:25:48,032 INFO [RS:0;d1d9e3766cb1:46057 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/d1d9e3766cb1,46057,1731212747651/d1d9e3766cb1%2C46057%2C1731212747651.1731212748027 2024-11-10T04:25:48,033 DEBUG [RS:0;d1d9e3766cb1:46057 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39207:39207),(127.0.0.1/127.0.0.1:44665:44665)] 2024-11-10T04:25:48,226 DEBUG [d1d9e3766cb1:36701 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:25:48,226 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:48,228 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,46057,1731212747651, state=OPENING 2024-11-10T04:25:48,230 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:25:48,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:48,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:48,232 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:25:48,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,46057,1731212747651}] 2024-11-10T04:25:48,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:25:48,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:25:48,272 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,33499,1731212565994/d1d9e3766cb1%2C33499%2C1731212565994.1731212566217 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:48,302 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40839/user/jenkins/test-data/dea947e2-97e7-860a-9e68-875a7a0056d6/WALs/d1d9e3766cb1,38007,1731212565036/d1d9e3766cb1%2C38007%2C1731212565036.meta.1731212565845.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T04:25:48,385 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:25:48,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52559, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:25:48,390 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:25:48,390 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:25:48,392 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C46057%2C1731212747651.meta, suffix=.meta, logDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/d1d9e3766cb1,46057,1731212747651, archiveDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/oldWALs, maxLogs=32 2024-11-10T04:25:48,392 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d1d9e3766cb1%2C46057%2C1731212747651.meta.1731212748392.meta 2024-11-10T04:25:48,397 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/d1d9e3766cb1,46057,1731212747651/d1d9e3766cb1%2C46057%2C1731212747651.meta.1731212748392.meta 2024-11-10T04:25:48,400 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44665:44665),(127.0.0.1/127.0.0.1:39207:39207)] 2024-11-10T04:25:48,404 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:25:48,404 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:25:48,404 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:25:48,405 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:25:48,405 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:25:48,405 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:25:48,405 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:25:48,405 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:25:48,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:25:48,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:25:48,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:48,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:48,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:25:48,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:25:48,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:48,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:48,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:25:48,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:25:48,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:48,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:48,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:25:48,409 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:25:48,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:25:48,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:25:48,410 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:25:48,410 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740 2024-11-10T04:25:48,411 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740 2024-11-10T04:25:48,412 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:25:48,412 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:25:48,412 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:25:48,414 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:25:48,414 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789163, jitterRate=0.003473535180091858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:25:48,414 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:25:48,415 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212748405Writing region info on filesystem at 1731212748405Initializing all the Stores at 1731212748405Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212748406 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212748406Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212748406Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212748406Cleaning up temporary data from old regions at 1731212748412 (+6 ms)Running coprocessor post-open hooks at 1731212748414 (+2 ms)Region opened successfully at 1731212748415 (+1 ms) 2024-11-10T04:25:48,416 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212748385 2024-11-10T04:25:48,418 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:25:48,418 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:25:48,418 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:48,419 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,46057,1731212747651, state=OPEN 2024-11-10T04:25:48,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:25:48,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:25:48,424 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:48,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:25:48,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:25:48,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:25:48,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,46057,1731212747651 in 192 msec 2024-11-10T04:25:48,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:25:48,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 603 msec 2024-11-10T04:25:48,429 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:25:48,429 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:25:48,430 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:25:48,431 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,46057,1731212747651, seqNum=-1] 2024-11-10T04:25:48,431 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:25:48,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50181, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:25:48,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 642 msec 2024-11-10T04:25:48,436 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212748436, completionTime=-1 2024-11-10T04:25:48,436 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:25:48,436 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:25:48,438 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:25:48,438 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212808438 2024-11-10T04:25:48,438 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731212868438 2024-11-10T04:25:48,438 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-10T04:25:48,438 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,36701,1731212747607-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:48,439 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,36701,1731212747607-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:48,439 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,36701,1731212747607-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:48,439 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:36701, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:48,439 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:48,439 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:48,440 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.761sec 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,36701,1731212747607-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:25:48,442 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,36701,1731212747607-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:25:48,444 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:25:48,444 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:25:48,444 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,36701,1731212747607-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:25:48,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1187e2f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:25:48,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,36701,-1 for getting cluster id 2024-11-10T04:25:48,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:25:48,466 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dcf74f63-ba5e-45e3-87c9-5523cf421b5a' 2024-11-10T04:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dcf74f63-ba5e-45e3-87c9-5523cf421b5a" 2024-11-10T04:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f2acbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:25:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,36701,-1] 2024-11-10T04:25:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:25:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:48,468 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:25:48,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321f0b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:25:48,469 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:25:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,46057,1731212747651, seqNum=-1] 2024-11-10T04:25:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:25:48,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:25:48,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:48,472 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:25:48,474 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T04:25:48,474 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T04:25:48,476 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/test.com,8080,1, archiveDir=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/oldWALs, maxLogs=32 2024-11-10T04:25:48,477 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731212748477 2024-11-10T04:25:48,481 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731212748477 2024-11-10T04:25:48,482 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39207:39207),(127.0.0.1/127.0.0.1:44665:44665)] 2024-11-10T04:25:48,483 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731212748483 2024-11-10T04:25:48,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,487 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731212748477 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731212748483 2024-11-10T04:25:48,488 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39207:39207),(127.0.0.1/127.0.0.1:44665:44665)] 2024-11-10T04:25:48,488 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731212748477 is not closed yet, will try archiving it next time 2024-11-10T04:25:48,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741835_1011 (size=93) 2024-11-10T04:25:48,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741835_1011 (size=93) 2024-11-10T04:25:48,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,490 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731212748477 to hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/oldWALs/test.com%2C8080%2C1.1731212748477 2024-11-10T04:25:48,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741836_1012 (size=93) 2024-11-10T04:25:48,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741836_1012 (size=93) 2024-11-10T04:25:48,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/oldWALs 2024-11-10T04:25:48,493 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731212748483) 2024-11-10T04:25:48,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:25:48,493 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:25:48,493 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:25:48,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:48,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:48,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:25:48,493 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:25:48,493 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=305067014, stopped=false 2024-11-10T04:25:48,494 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,36701,1731212747607 2024-11-10T04:25:48,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:25:48,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:25:48,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:48,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:48,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:25:48,495 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:25:48,495 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:25:48,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:48,496 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,46057,1731212747651' ***** 2024-11-10T04:25:48,496 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:25:48,496 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:25:48,496 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:25:48,496 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:25:48,496 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:25:48,496 INFO [RS:0;d1d9e3766cb1:46057 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:25:48,496 INFO [RS:0;d1d9e3766cb1:46057 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:25:48,496 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:48,496 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:25:48,496 INFO [RS:0;d1d9e3766cb1:46057 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:46057. 2024-11-10T04:25:48,496 DEBUG [RS:0;d1d9e3766cb1:46057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:25:48,496 DEBUG [RS:0;d1d9e3766cb1:46057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:48,497 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:25:48,497 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:25:48,497 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:25:48,497 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:25:48,497 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T04:25:48,497 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T04:25:48,497 DEBUG [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:25:48,497 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:25:48,497 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:25:48,497 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:25:48,497 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:25:48,497 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:25:48,497 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-10T04:25:48,513 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740/.tmp/ns/a114cfcc6f3a4aecab27d8add5ceea69 is 43, key is default/ns:d/1731212748432/Put/seqid=0 2024-11-10T04:25:48,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741837_1013 (size=5153) 2024-11-10T04:25:48,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741837_1013 (size=5153) 2024-11-10T04:25:48,518 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740/.tmp/ns/a114cfcc6f3a4aecab27d8add5ceea69 2024-11-10T04:25:48,522 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740/.tmp/ns/a114cfcc6f3a4aecab27d8add5ceea69 as hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740/ns/a114cfcc6f3a4aecab27d8add5ceea69 2024-11-10T04:25:48,527 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740/ns/a114cfcc6f3a4aecab27d8add5ceea69, entries=2, sequenceid=6, filesize=5.0 K 2024-11-10T04:25:48,528 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false 2024-11-10T04:25:48,528 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:25:48,531 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T04:25:48,532 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:25:48,532 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:25:48,532 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212748497Running coprocessor pre-close hooks at 1731212748497Disabling compacts and flushes for region at 1731212748497Disabling writes for close at 1731212748497Obtaining lock to block concurrent updates at 1731212748497Preparing flush snapshotting stores in 1588230740 at 1731212748497Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731212748497Flushing stores of hbase:meta,,1.1588230740 at 1731212748498 (+1 ms)Flushing 1588230740/ns: creating writer at 1731212748498Flushing 1588230740/ns: appending metadata at 1731212748512 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731212748512Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e76215f: reopening flushed file at 1731212748522 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false at 1731212748528 (+6 ms)Writing region close event to WAL at 1731212748529 (+1 ms)Running coprocessor post-close hooks at 1731212748532 (+3 ms)Closed at 1731212748532 2024-11-10T04:25:48,532 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:25:48,697 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,46057,1731212747651; all regions closed. 2024-11-10T04:25:48,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741834_1010 (size=1152) 2024-11-10T04:25:48,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741834_1010 (size=1152) 2024-11-10T04:25:48,702 DEBUG [RS:0;d1d9e3766cb1:46057 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/oldWALs 2024-11-10T04:25:48,702 INFO [RS:0;d1d9e3766cb1:46057 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C46057%2C1731212747651.meta:.meta(num 1731212748392) 2024-11-10T04:25:48,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741833_1009 (size=93) 2024-11-10T04:25:48,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741833_1009 (size=93) 2024-11-10T04:25:48,706 DEBUG [RS:0;d1d9e3766cb1:46057 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/oldWALs 2024-11-10T04:25:48,707 INFO [RS:0;d1d9e3766cb1:46057 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d1d9e3766cb1%2C46057%2C1731212747651:(num 1731212748027) 2024-11-10T04:25:48,707 DEBUG [RS:0;d1d9e3766cb1:46057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:25:48,707 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:25:48,707 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:25:48,707 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:25:48,707 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:25:48,707 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:25:48,707 INFO [RS:0;d1d9e3766cb1:46057 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46057 2024-11-10T04:25:48,709 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:25:48,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,46057,1731212747651 2024-11-10T04:25:48,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:25:48,711 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,46057,1731212747651] 2024-11-10T04:25:48,713 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,46057,1731212747651 already deleted, retry=false 2024-11-10T04:25:48,713 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,46057,1731212747651 expired; onlineServers=0 2024-11-10T04:25:48,713 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,36701,1731212747607' ***** 2024-11-10T04:25:48,713 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:25:48,713 INFO [M:0;d1d9e3766cb1:36701 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:25:48,713 INFO [M:0;d1d9e3766cb1:36701 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:25:48,713 DEBUG [M:0;d1d9e3766cb1:36701 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:25:48,713 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:25:48,713 DEBUG [M:0;d1d9e3766cb1:36701 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:25:48,713 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212747798 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731212747798,5,FailOnTimeoutGroup] 2024-11-10T04:25:48,713 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212747798 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731212747798,5,FailOnTimeoutGroup] 2024-11-10T04:25:48,713 INFO [M:0;d1d9e3766cb1:36701 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:25:48,713 INFO [M:0;d1d9e3766cb1:36701 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:25:48,714 DEBUG [M:0;d1d9e3766cb1:36701 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:25:48,714 INFO [M:0;d1d9e3766cb1:36701 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:25:48,714 INFO [M:0;d1d9e3766cb1:36701 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:25:48,714 INFO [M:0;d1d9e3766cb1:36701 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:25:48,714 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:25:48,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:25:48,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:25:48,715 DEBUG [M:0;d1d9e3766cb1:36701 {}] zookeeper.ZKUtil(347): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:25:48,716 WARN [M:0;d1d9e3766cb1:36701 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:25:48,716 INFO [M:0;d1d9e3766cb1:36701 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/.lastflushedseqids 2024-11-10T04:25:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741838_1014 (size=99) 2024-11-10T04:25:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741838_1014 (size=99) 2024-11-10T04:25:48,721 INFO [M:0;d1d9e3766cb1:36701 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:25:48,722 INFO [M:0;d1d9e3766cb1:36701 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:25:48,722 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:25:48,722 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:48,722 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:48,722 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:25:48,722 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:48,722 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-10T04:25:48,737 DEBUG [M:0;d1d9e3766cb1:36701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa1beab9702f4386b760053f37f2a4a9 is 82, key is hbase:meta,,1/info:regioninfo/1731212748418/Put/seqid=0 2024-11-10T04:25:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741839_1015 (size=5672) 2024-11-10T04:25:48,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741839_1015 (size=5672) 2024-11-10T04:25:48,742 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa1beab9702f4386b760053f37f2a4a9 2024-11-10T04:25:48,761 DEBUG [M:0;d1d9e3766cb1:36701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/832ab6d2363442928d795dae35d06624 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731212748436/Put/seqid=0 2024-11-10T04:25:48,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741840_1016 (size=5275) 2024-11-10T04:25:48,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741840_1016 (size=5275) 2024-11-10T04:25:48,768 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/832ab6d2363442928d795dae35d06624 2024-11-10T04:25:48,786 DEBUG [M:0;d1d9e3766cb1:36701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75938b7f471446ceb19ae882c3e4bfe4 is 69, key is d1d9e3766cb1,46057,1731212747651/rs:state/1731212747885/Put/seqid=0 2024-11-10T04:25:48,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741841_1017 (size=5156) 2024-11-10T04:25:48,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741841_1017 (size=5156) 2024-11-10T04:25:48,790 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75938b7f471446ceb19ae882c3e4bfe4 2024-11-10T04:25:48,807 DEBUG [M:0;d1d9e3766cb1:36701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/df2ce014d4ff408a877242064f8cbf59 is 52, key is load_balancer_on/state:d/1731212748474/Put/seqid=0 2024-11-10T04:25:48,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:48,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46057-0x101906d98b20001, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:48,811 INFO [RS:0;d1d9e3766cb1:46057 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:25:48,811 INFO [RS:0;d1d9e3766cb1:46057 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,46057,1731212747651; zookeeper connection closed. 2024-11-10T04:25:48,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741842_1018 (size=5056) 2024-11-10T04:25:48,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741842_1018 (size=5056) 2024-11-10T04:25:48,812 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@783150c1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@783150c1 2024-11-10T04:25:48,812 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:25:48,812 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/df2ce014d4ff408a877242064f8cbf59 2024-11-10T04:25:48,816 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa1beab9702f4386b760053f37f2a4a9 as hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa1beab9702f4386b760053f37f2a4a9 2024-11-10T04:25:48,821 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa1beab9702f4386b760053f37f2a4a9, entries=8, sequenceid=29, filesize=5.5 K 2024-11-10T04:25:48,822 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/832ab6d2363442928d795dae35d06624 as hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/832ab6d2363442928d795dae35d06624 2024-11-10T04:25:48,826 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/832ab6d2363442928d795dae35d06624, entries=3, sequenceid=29, filesize=5.2 K 2024-11-10T04:25:48,827 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75938b7f471446ceb19ae882c3e4bfe4 as hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/75938b7f471446ceb19ae882c3e4bfe4 2024-11-10T04:25:48,831 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/75938b7f471446ceb19ae882c3e4bfe4, entries=1, sequenceid=29, filesize=5.0 K 2024-11-10T04:25:48,832 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/df2ce014d4ff408a877242064f8cbf59 as hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/df2ce014d4ff408a877242064f8cbf59 2024-11-10T04:25:48,836 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37439/user/jenkins/test-data/18a73f51-878d-b848-523f-a043135b67a4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/df2ce014d4ff408a877242064f8cbf59, entries=1, sequenceid=29, filesize=4.9 K 2024-11-10T04:25:48,837 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=29, compaction requested=false 2024-11-10T04:25:48,839 INFO [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:25:48,839 DEBUG [M:0;d1d9e3766cb1:36701 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212748722Disabling compacts and flushes for region at 1731212748722Disabling writes for close at 1731212748722Obtaining lock to block concurrent updates at 1731212748722Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212748722Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731212748722Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212748723 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212748723Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212748737 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212748737Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212748746 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212748760 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212748760Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212748772 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212748785 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212748785Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731212748794 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731212748807 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731212748807Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4af96ebd: reopening flushed file at 1731212748816 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2686dcd5: reopening flushed file at 1731212748821 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42884d2e: reopening flushed file at 1731212748827 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55c83b7c: reopening flushed file at 1731212748831 (+4 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=29, compaction requested=false at 1731212748837 (+6 ms)Writing region close event to WAL at 1731212748839 (+2 ms)Closed at 1731212748839 2024-11-10T04:25:48,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T04:25:48,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43087 is added to blk_1073741830_1006 (size=10311) 2024-11-10T04:25:48,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40157 is added to blk_1073741830_1006 (size=10311) 2024-11-10T04:25:48,842 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:25:48,842 INFO [M:0;d1d9e3766cb1:36701 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:25:48,842 INFO [M:0;d1d9e3766cb1:36701 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36701 2024-11-10T04:25:48,843 INFO [M:0;d1d9e3766cb1:36701 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:25:48,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:48,945 INFO [M:0;d1d9e3766cb1:36701 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:25:48,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36701-0x101906d98b20000, quorum=127.0.0.1:55780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:25:48,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4cb9612e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:25:48,948 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@186f146f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:25:48,948 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:25:48,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32c86c86{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:25:48,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47b9368{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.log.dir/,STOPPED} 2024-11-10T04:25:48,949 WARN [BP-1332666732-172.17.0.2-1731212746965 heartbeating to localhost/127.0.0.1:37439 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:25:48,949 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:25:48,949 WARN [BP-1332666732-172.17.0.2-1731212746965 heartbeating to localhost/127.0.0.1:37439 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1332666732-172.17.0.2-1731212746965 (Datanode Uuid 73d2af8d-243a-4b02-90dc-952d0380b620) service to localhost/127.0.0.1:37439 2024-11-10T04:25:48,949 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:25:48,950 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data3/current/BP-1332666732-172.17.0.2-1731212746965 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:48,950 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data4/current/BP-1332666732-172.17.0.2-1731212746965 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:48,950 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:25:48,952 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@61266769{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:25:48,952 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49428dc3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:25:48,952 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:25:48,952 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41f14207{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:25:48,952 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6162294b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.log.dir/,STOPPED} 2024-11-10T04:25:48,953 WARN [BP-1332666732-172.17.0.2-1731212746965 heartbeating to localhost/127.0.0.1:37439 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:25:48,953 WARN [BP-1332666732-172.17.0.2-1731212746965 heartbeating to localhost/127.0.0.1:37439 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1332666732-172.17.0.2-1731212746965 (Datanode Uuid d78f89ed-9edf-4e79-b4d0-d944a642d45a) service to localhost/127.0.0.1:37439 2024-11-10T04:25:48,953 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:25:48,954 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:25:48,954 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data1/current/BP-1332666732-172.17.0.2-1731212746965 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:48,954 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/cluster_05160ef8-6de1-7b51-ce20-c5c299b6e417/data/data2/current/BP-1332666732-172.17.0.2-1731212746965 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:25:48,954 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:25:48,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b418861{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:25:48,960 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7cdb81da{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:25:48,960 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:25:48,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1cb0de45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:25:48,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5031c9de{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59db72f9-1a2b-c827-f276-588a01332747/hadoop.log.dir/,STOPPED} 2024-11-10T04:25:48,966 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:25:48,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:25:48,989 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 230) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37439 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:37439 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: globalEventExecutor-1-22 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113) app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37439 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37439 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37439 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37439 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37439 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37439 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=537 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=41 (was 41), ProcessCount=11 (was 11), AvailableMemoryMB=6347 (was 6357)